From f50fa34d6d3c1559849b8b2ed00ea99259e6dd94 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Mon, 6 Nov 2023 13:45:16 +0000 Subject: [PATCH 01/45] Created model classes for SDK python tests --- app/chip_tool/chip_tool.py | 12 +- app/core/config.py | 2 +- .../yaml_tests/test_sdk_yaml_collection.py | 1 + app/tests/yaml_tests/test_yaml_folder.py | 8 +- test_collections/sdk_tests/__init__.py | 1 + .../support/python_testing/__init__.py | 22 ++ .../support/python_testing/models/__init__.py | 17 ++ .../models/python_test_folder.py | 56 +++++ .../models/python_test_models.py | 48 +++++ .../models/python_test_parser.py | 151 +++++++++++++ .../python_testing/models/test_case.py | 200 ++++++++++++++++++ .../models/test_declarations.py | 63 ++++++ .../python_testing/models/test_suite.py | 96 +++++++++ .../python_testing/sdk_python_test_tests.py | 104 +++++++++ .../support/yaml_tests/sdk_yaml_tests.py | 5 +- 15 files changed, 775 insertions(+), 11 deletions(-) create mode 100644 test_collections/sdk_tests/support/python_testing/__init__.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/__init__.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_test_folder.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_test_models.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_test_parser.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/test_case.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/test_declarations.py create mode 100644 test_collections/sdk_tests/support/python_testing/models/test_suite.py create mode 100644 test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 6e9d1e33..8201c25e 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -99,7 +99,9 @@ # Websocket runner BACKEND_ROOT = Path(__file__).parents[2] -TEST_COLLECTION_SDK_CHECKOUT_PATH = BACKEND_ROOT / Path("test_collections/sdk_tests/sdk_checkout/") +TEST_COLLECTION_SDK_CHECKOUT_PATH = BACKEND_ROOT / Path( + "test_collections/sdk_tests/sdk_checkout/" +) YAML_TESTS_PATH_BASE = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path("yaml_tests/") YAML_TESTS_PATH = YAML_TESTS_PATH_BASE / Path("yaml/sdk") XML_SPEC_DEFINITION_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path( @@ -130,6 +132,7 @@ class ChipToolUnknownTestType(Exception): class ChipToolTestType(str, Enum): CHIP_TOOL = "chip-tool" CHIP_APP = "chip-app" + PTYHON_TEST = "python-test" class ChipTool(metaclass=Singleton): @@ -263,6 +266,8 @@ async def start_chip_server( elif test_type == ChipToolTestType.CHIP_APP: prefix = CHIP_APP_EXE command = ["--interactive", "--port 9002"] + elif test_type == ChipToolTestType.PTYHON_TEST: + pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") @@ -578,7 +583,7 @@ async def run_test( if test_type == ChipToolTestType.CHIP_TOOL: test_path = f"{YAML_TESTS_PATH}/{test_id}.yaml" - else: + elif test_type == ChipToolTestType.CHIP_APP: test_path = f"{YAML_TESTS_PATH}/{test_id}_Simulated.yaml" parser_config = TestParserConfig(pics_path, self.specifications, test_options) @@ -590,6 +595,9 @@ async def run_test( adapter = ChipToolAdapter.Adapter(parser_config.definitions) elif test_type == ChipToolTestType.CHIP_APP: adapter = ChipAppAdapter.Adapter(parser_config.definitions) + elif test_type == ChipToolTestType.PTYHON_TEST: + # TODO - run_test for python_test must be updated + pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") diff --git a/app/core/config.py b/app/core/config.py index ce41af1a..c6dd5ee5 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -90,7 +90,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: # Logging LOGGING_PATH: str = "./logs" LOGGING_FILENAME: str = "{time:YYYY-MM-DD}.log" - LOGGING_LEVEL: str = "debug" + LOGGING_LEVEL: str = "info" LOGGING_ROTATION: str = "20 days" LOGGING_RETENTION: str = "1 months" LOGGING_FORMAT: str = ( diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index d5891120..db81d4cc 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -34,6 +34,7 @@ VERSION_FILE_FILENAME = ".version" VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") + @pytest.fixture def yaml_collection() -> YamlCollectionDeclaration: test_sdk_yaml_path = Path(__file__).parent / "test_yamls" diff --git a/app/tests/yaml_tests/test_yaml_folder.py b/app/tests/yaml_tests/test_yaml_folder.py index 81f77273..65a5012e 100644 --- a/app/tests/yaml_tests/test_yaml_folder.py +++ b/app/tests/yaml_tests/test_yaml_folder.py @@ -31,9 +31,7 @@ def test_yaml_folder_version() -> None: with mock.patch( "test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder.open", new=mock.mock_open(read_data=version_file_content), - ), mock.patch.object( - target=Path, attribute="exists", return_value=True - ) as _: + ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: yaml_folder = YamlTestFolder(test_yaml_path) assert yaml_folder.version == version_file_content @@ -41,9 +39,7 @@ def test_yaml_folder_version() -> None: def test_yaml_folder_version_missing() -> None: expected_version = "Unknown" - with mock.patch.object( - target=Path, attribute="exists", return_value=False - ) as _: + with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: yaml_folder = YamlTestFolder(test_yaml_path) assert yaml_folder.version == expected_version diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index ec9f4b95..612e533e 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -14,3 +14,4 @@ # limitations under the License. # from .support.yaml_tests import sdk_collection +from .support.python_testing import sdk_python_collection diff --git a/test_collections/sdk_tests/support/python_testing/__init__.py b/test_collections/sdk_tests/support/python_testing/__init__.py new file mode 100644 index 00000000..31921266 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/__init__.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from app.test_engine.models.test_declarations import TestCollectionDeclaration + +from .sdk_python_test_tests import sdk_python_test_collection + +# Test engine will auto load TestCollectionDeclarations declared inside the package +# initializer +sdk_python_collection: TestCollectionDeclaration = sdk_python_test_collection() diff --git a/test_collections/sdk_tests/support/python_testing/models/__init__.py b/test_collections/sdk_tests/support/python_testing/models/__init__.py new file mode 100644 index 00000000..8b47fb7f --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .test_case import PythonTestCase +from .test_suite import SuiteType, PythonTestSuite diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py new file mode 100644 index 00000000..f8fb31a0 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +UNKNOWN_version = "Unknown" +VERSION_FILE_FILENAME = ".version" +VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") + +class PythonTestFolder: + """Representing a folder with Test Python files. + + Note: Python version is read from .version file in folder on init. + """ + + def __init__(self, path: Path, filename_pattern: str = "*") -> None: + self.path = path + self.filename_pattern = filename_pattern + self.version = self.__version() + + def __version(self) -> str: + """Read version string from .version file in + /app/backend/test_collections/sdk_tests/sdk_checkout path.""" + version_file_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME + + if not version_file_path.exists(): + return UNKNOWN_version + else: + with open(version_file_path, "r") as file: + return file.read().rstrip() + + def python_file_paths(self) -> list[Path]: + """Get list of paths to Python test files in folder. + + Filename filter can be applied if only some files should be selected. + Note: filter is without extension. Will search for .py files + + Args: + filename_pattern (str, optional): custom file filter. Defaults to "*". + + Returns: + list[Path]: list of paths to Python test files. + """ + return list(self.path.glob(self.filename_pattern + ".py")) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py new file mode 100644 index 00000000..d3246fd3 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import Enum +from pathlib import Path +from typing import Any, Optional + +from pydantic import BaseModel, Field + +### +# This file declares Python test models that are used to parse the Python Test Cases. +### + + +class PythonTestType(Enum): + AUTOMATED = 0 + SEMI_AUTOMATED = 1 + SIMULATED = 2 + + +class PythonTestStep(BaseModel): + label: str + PICS: Optional[str] = None + verification: Optional[str] = None + command: Optional[str] + disabled: bool = False + arguments: Optional[dict[str, Any]] + + +class PythonTest(BaseModel): + name: str + PICS: set[str] = set() + config: dict[str, Any] + steps: list[PythonTestStep] = Field(alias="tests") + type: PythonTestType = PythonTestType.AUTOMATED + path: Optional[Path] diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py new file mode 100644 index 00000000..9b3ea440 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -0,0 +1,151 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from typing import List +import ast +from loguru import logger +from pydantic import ValidationError + +from .python_test_models import PythonTest, PythonTestType, PythonTestStep + + +class PythonParserException(Exception): + """Raised when an error occurs during the parser of python file.""" + + +def _test_type(test: PythonTest) -> PythonTestType: + """Determine the type of a test based on the parsed python. + + This is mainly determined by the number of disabled test steps. + + Args: + test (PythonTest): parsed python test + + Returns: + TestType: + - Manual: All steps disabled + - Semi-Automated: some steps are disabled + - Automated: no disabled steps + - Simulated: Tests where file name have "Simulated" + """ + if test.path is not None and "Simulated" in str(test.path): + return PythonTestType.SIMULATED + + steps = test.steps + + # if any step has a UserPrompt, categorize as semi-automated + if any(s.command == "UserPrompt" for s in steps): + return PythonTestType.SEMI_AUTOMATED + + # Otherwise Automated + return PythonTestType.AUTOMATED + + +def parse_python_test(path: Path) -> PythonTest: + """Parse a single Python test file into PythonTest model. + + This will also annotate parsed python test with it's path and test type. + """ + python_steps: list[PythonTestStep] = [] + tc_pics = [] + tc_config = {} + + + if "ACE_1_3" in str(path): + python_steps = tc_steps(path) + tc_desc = tc_description(path) + # with open(str(path)) as python_file: + # parsed = ast.parse(python_file.read()) + + # functions = [n for n in parsed.body if isinstance(n, ast.FunctionDef)] + # classes = [n for n in parsed.body if isinstance(n, ast.ClassDef)] + + # print(functions) + # print(classes) + + # for function in functions: + # result.append(show_info(function)) + + # for step in steps: + # python_steps = tc_steps(path) + # tc_desc = tc_description(path) + + + return PythonTest( + name=tc_desc, tests=python_steps, config=tc_config, PICS=tc_pics + ) + +def tc_description(path: Path) -> str: + with open(str(path)) as python_file: + parsed = ast.parse(python_file.read()) + + # functions = [n for n in parsed.body if isinstance(n, ast.FunctionDef)] + classes = [n for n in parsed.body if isinstance(n, ast.ClassDef)] + + # for function in functions: + # print(function) + # result.append(show_info(function)) + + tc_desc = "'" + python_steps:List[PythonTestStep] = [] + + for class_ in classes: + methods = [n for n in class_.body if isinstance(n, ast.FunctionDef)] + for method in methods: + if "desc_" in method.name: + tc_desc = method.body[0].value.value + elif "steps_" in method.name: + python_steps = retrieve_steps(method) + # desc_TC_ACE_1_3 -> method.body[0].value.value + # method.name + # print(method) + # result.append((class_.name + "." + show_info(method))) + # # desc_* + # method.body[0].value.value + # # steps_* + # method.body[0].value.elts[4].args[1].value + + # method.body[0].value.elts[0].keywords[0].arg + # method.body[0].value.elts[0].keywords[0].value.value + + # print(", ".join(result)) + + return "" + +def retrieve_steps(method: ast.FunctionDef)-> List[PythonTestStep]: + python_steps:List[PythonTestStep] = [] + for step in method.body[0].value.elts: + step_name = step.args[1].value + step_is_commissioning = False + if step.keywords and 'is_commissioning' in step.keywords[0].arg: + is_commissioning = step.keywords[0].value.value + + python_steps.append( + PythonTestStep( + label=step_name, is_commissioning=step_is_commissioning + ) + ) + + return python_steps + +# def tc_steps(path: Path) -> List[PythonTestStep]: +# python_steps:List[PythonTestStep] = [] +# python_steps.append( +# PythonTestStep( +# label="step.description", is_commissioning=False # step.is_commissioning +# ) +# ) +# return python_steps diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py new file mode 100644 index 00000000..3c9d6149 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -0,0 +1,200 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import re +from typing import Any, Type, TypeVar + +from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.test_case import ChipToolManualPromptTest, ChipToolTest +from app.test_engine.logger import test_engine_logger +from app.test_engine.models import ( + ManualTestCase, + ManualVerificationTestStep, + TestCase, + TestStep, +) + +from .python_test_models import PythonTest, PythonTestStep, PythonTestType + +# Custom type variable used to annotate the factory method in PythonTestCase. +T = TypeVar("T", bound="PythonTestCase") + + +class PythonTestCase(TestCase): + """Base class for all Python based test cases. + + This class provides a class factory that will dynamically declare a new sub-class + based on the test-type the Python test is expressing. + + The PythonTest will be stored as a class property that will be used at run-time in all + instances of such subclass. + """ + + python_test: PythonTest + python_test_version: str + + @classmethod + def pics(cls) -> set[str]: + """Test Case level PICS. Read directly from parsed python test.""" + return cls.python_test.PICS + + @classmethod + def default_test_parameters(cls) -> dict[str, Any]: + """Python test config dict, sometimes have a nested dict with type and default value. + Only defaultValue is used in this case. + """ + parameters = {} + for param_name, value in cls.python_test.config.items(): + if isinstance(value, dict): + if "defaultValue" in value: + parameters[param_name] = value["defaultValue"] + else: + parameters[param_name] = value + return parameters + + async def setup(self) -> None: + """Override Setup to log Python Test version.""" + test_engine_logger.info(f"Python Test Version: {self.python_test_version}") + try: + await super().setup() + except NotImplementedError: + pass + + @classmethod + def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: + """Dynamically declares a subclass based on the type of Python test.""" + case_class: Type[PythonTestCase] + if test.type == PythonTestType.SEMI_AUTOMATED: + case_class = PythonSemiAutomatedChipToolTestCase + elif test.type == PythonTestType.SIMULATED: + case_class = PythonSimulatedTestCase + else: # Automated + case_class = PythonChipToolTestCase + + return case_class.__class_factory(test=test, python_test_version=python_test_version) + + @classmethod + def __class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: + """Common class factory method for all subclasses of PythonTestCase.""" + identifier = cls.__test_identifier(test.name) + class_name = cls.__class_name(identifier) + title = cls.__title(identifier=identifier, test_python=test) + + return type( + class_name, + (cls,), + { + "python_test": test, + "python_test_version": python_test_version, + "chip_tool_test_identifier": class_name, + "metadata": { + "public_id": identifier, + "version": "0.0.1", + "title": title, + "description": test.name, + }, + }, + ) + + @staticmethod + def __test_identifier(name: str) -> str: + """Find TC-XX-1.1 in Python test title. + Note some have [TC-XX-1.1] and others TC-XX-1.1 + """ + title_pattern = re.compile(r"(?PTC-[^\s\]]*)") + if match := re.search(title_pattern, name): + return match["title"] + else: + return name + + @staticmethod + def __class_name(identifier: str) -> str: + """Replace all non-alphanumeric characters with _ to make valid class name.""" + return re.sub("[^0-9a-zA-Z]+", "_", identifier) + + @staticmethod + def __has_steps_disabled(test_python: PythonTest) -> bool: + """If some but not all steps are disabled, return true. False otherwise.""" + len_disabled_steps = len([s for s in test_python.steps if not s.disabled]) + + if len_disabled_steps == 0: + return False + else: + return len_disabled_steps < len(test_python.steps) + + @classmethod + def __title(cls, identifier: str, test_python: PythonTest) -> str: + """Annotate Title with Semi-automated and Steps Disabled tests in the test + title. + """ + title = identifier + + if test_python.type == PythonTestType.SEMI_AUTOMATED: + title += " (Semi-automated)" + + if cls.__has_steps_disabled(test_python): + title += " (Steps Disabled)" + + return title + + def _append_automated_test_step(self, python_test_step: PythonTestStep) -> None: + """ + Disabled steps are ignored. + (Such tests will be marked as 'Steps Disabled' elsewhere) + + UserPrompt are special cases that will prompt test operator for input. + """ + if python_test_step.disabled: + test_engine_logger.info( + f"{self.public_id()}: skipping disabled step: {python_test_step.label}" + ) + return + + step = TestStep(python_test_step.label) + if python_test_step.command == "UserPrompt": + step = ManualVerificationTestStep( + name=python_test_step.label, + verification=python_test_step.verification, + ) + + self.test_steps.append(step) + + +class PythonChipToolTestCase(PythonTestCase, ChipToolTest): + """Automated test cases using chip-tool.""" + + test_type = ChipToolTestType.CHIP_TOOL + + def create_test_steps(self) -> None: + self.test_steps = [TestStep("Start chip-tool test")] + for step in self.python_test.steps: + self._append_automated_test_step(step) + + +class PythonSemiAutomatedChipToolTestCase(PythonChipToolTestCase, ChipToolManualPromptTest): + """Semi-Automated test cases, need special step for users to attach logs + for manual steps, so inheriting from ChipToolManualPromptTest. + """ + + +class PythonSimulatedTestCase(PythonTestCase, ChipToolTest): + """Simulated test cases using chip-app""" + + test_type = ChipToolTestType.CHIP_APP + + def create_test_steps(self) -> None: + self.test_steps = [TestStep("Start chip-app test")] + for step in self.python_test.steps: + self._append_automated_test_step(step) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py new file mode 100644 index 00000000..1acd7363 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Type + +from app.test_engine.models.test_declarations import ( + TestCaseDeclaration, + TestCollectionDeclaration, + TestSuiteDeclaration, +) + +from .test_case import PythonTestCase +from .test_suite import SuiteType, PythonTestSuite +from .python_test_folder import PythonTestFolder +from .python_test_models import PythonTest, PythonTestType + + +class PythonCollectionDeclaration(TestCollectionDeclaration): + def __init__(self, folder: PythonTestFolder, name: str) -> None: + super().__init__(path=str(folder.path), name=name) + self.python_test_version = folder.version + + +class PythonSuiteDeclaration(TestSuiteDeclaration): + """Direct initialization for Python Test Suite.""" + + class_ref: Type[PythonTestSuite] + + def __init__(self, name: str, suite_type: SuiteType, version: str) -> None: + super().__init__( + PythonTestSuite.class_factory( + name=name, + suite_type=suite_type, + python_test_version=version, + ) + ) + + +class PythonCaseDeclaration(TestCaseDeclaration): + """Direct initialization for Python Test Case.""" + + class_ref: Type[PythonTestCase] + + def __init__(self, test: PythonTest, python_test_version: str) -> None: + super().__init__( + PythonTestCase.class_factory(test=test, python_test_version=python_test_version) + ) + + @property + def test_type(self) -> PythonTestType: + return self.class_ref.python_test.type diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py new file mode 100644 index 00000000..ff096fac --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import Enum +from typing import Type, TypeVar + +from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.test_suite import ChipToolSuite +from app.test_engine.logger import test_engine_logger as logger +from app.test_engine.models import TestSuite + + +class PythonTestSuiteFactoryError(Exception): + pass + + +class SuiteType(Enum): + SIMULATED = 1 + AUTOMATED = 2 + + +# Custom Type variable used to annotate the factory methods of classmethod. +T = TypeVar("T", bound="PythonTestSuite") + + +class PythonTestSuite(TestSuite): + """Base class for all Python tests based test suites. + + This class provides a class factory that will dynamically declare a new sub-class + based on the suite-type. + """ + + python_test_version: str + suite_name: str + + async def setup(self) -> None: + """Override Setup to log Python Test version.""" + logger.info(f"Python Test Version: {self.python_test_version}") + + @classmethod + def class_factory( + cls, suite_type: SuiteType, name: str, python_test_version: str + ) -> Type[T]: + """Dynamically declares a subclass based on the type of test suite.""" + suite_class = PythonTestSuite + + if suite_type == SuiteType.SIMULATED: + suite_class = SimulatedPythonTestSuite + elif suite_type == SuiteType.AUTOMATED: + suite_class = ChipToolPythonTestSuite + + return suite_class.__class_factory(name=name, python_test_version=python_test_version) + + @classmethod + def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: + """Common class factory method for all subclasses of PythonTestSuite.""" + + return type( + name, + (cls,), + { + "name": name, + "python_test_version": python_test_version, + "metadata": { + "public_id": name, + "version": "0.0.1", + "title": name, + "description": name, + }, + }, + ) + + +class ChipToolPythonTestSuite(PythonTestSuite, ChipToolSuite): + test_type = ChipToolTestType.CHIP_TOOL + + async def setup(self) -> None: + """Due top multi inheritance, we need to call setup on both super classes.""" + await PythonTestSuite.setup(self) + await ChipToolSuite.setup(self) + + +class SimulatedPythonTestSuite(ChipToolPythonTestSuite): + test_type = ChipToolTestType.CHIP_APP diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py new file mode 100644 index 00000000..a57dc024 --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +from loguru import logger + +from .models.test_declarations import ( + PythonCaseDeclaration, + PythonCollectionDeclaration, + PythonSuiteDeclaration, +) +from .models.test_suite import SuiteType +from .models.python_test_folder import PythonTestFolder +from .models.python_test_models import PythonTestType +from .models.python_test_parser import PythonParserException, parse_python_test + +### +# This file hosts logic load and parse Python test-cases, located in +# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. The `sdk` sub-folder here is automatically +# maintained using the `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. +# +# The Python Tests are organized into 2 Test Suites: +# - Automated and Semi-Automated using Chip-Tool +# - Simulated using Chip-App1 +### + +SDK_PYTHON_TEST_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk") +SDK_PYTHON_TEST_FOLDER = PythonTestFolder(path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*") + +def _init_test_suites(python_test_version: str) -> dict[SuiteType, PythonSuiteDeclaration]: + return { + SuiteType.AUTOMATED: PythonSuiteDeclaration( + name="Python Testing Suite", + suite_type=SuiteType.AUTOMATED, + version=python_test_version, + ), + } + + +def _parse_python_test_to_test_case_declaration( + python_test_path: Path, python_test_version: str +) -> PythonCaseDeclaration: + python_test = parse_python_test(python_test_path) + return PythonCaseDeclaration(test=python_test, python_test_version=python_test_version) + + +def _parse_all_sdk_python_tests( + python_test_files: list[Path], python_test_version: str +) -> list[PythonSuiteDeclaration]: + """Parse all python test files and organize them in the 3 test suites: + - Automated and Semi-Automated using Chip-Tool + - Simulated using Chip-App1 + - Manual + """ + suites = _init_test_suites(python_test_version) + + for python_test_file in python_test_files: + try: + test_case = _parse_python_test_to_test_case_declaration( + python_test_path=python_test_file, python_test_version=python_test_version + ) + + if test_case.test_type == PythonTestType.SIMULATED: + suites[SuiteType.SIMULATED].add_test_case(test_case) + else: + suites[SuiteType.AUTOMATED].add_test_case(test_case) + except PythonParserException: + # If an exception was raised during parse process, the python file will be + # ignored and the loop will continue with the next python file + logger.error(f"Error while parsing Python File: {python_test_file}") + + return list(suites.values()) + + +def sdk_python_test_collection( + python_test_folder: PythonTestFolder = SDK_PYTHON_TEST_FOLDER, +) -> PythonCollectionDeclaration: + """Declare a new collection of test suites with the 3 test suites.""" + collection = PythonCollectionDeclaration( + name="SDK Python Tests", folder=python_test_folder + ) + + files = python_test_folder.python_file_paths() + version = python_test_folder.version + suites = _parse_all_sdk_python_tests(python_test_files=files, python_test_version=version) + + for suite in suites: + suite.sort_test_cases() + collection.add_test_suite(suite) + + return collection diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index 91ea69e8..12cb62f5 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -29,8 +29,9 @@ ### # This file hosts logic load and parse YAML test-cases, located in -# `test_collections/yaml_tests/yaml/sdk`. The `sdk` sub-folder here is automatically -# maintained using the `scripts/fetch_sdk_yaml_tests_and_runner.sh` script. +# `test_collections/sdk_tests/sdk_checkout/yaml_tests/yaml/sdk`. +# The `sdk` sub-folder here is automatically maintained using the +# `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. # # The YAML Tests are organized into 3 Test Suites: # - Automated and Semi-Automated using Chip-Tool From 6a772650ab5b12703dde02a316b2990f0632a8d1 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 12:23:20 +0000 Subject: [PATCH 02/45] Added parser implementation to python test --- app/chip_tool/chip_tool.py | 8 +- test_collections/sdk_tests/__init__.py | 2 +- .../support/python_testing/models/__init__.py | 2 +- .../models/python_test_folder.py | 3 +- .../models/python_test_models.py | 2 - .../models/python_test_parser.py | 151 ++++++------------ .../python_testing/models/test_case.py | 62 +------ .../models/test_declarations.py | 8 +- .../python_testing/models/test_suite.py | 17 +- .../python_testing/sdk_python_test_tests.py | 40 +++-- 10 files changed, 99 insertions(+), 196 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 8201c25e..038d9e8a 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -132,7 +132,7 @@ class ChipToolUnknownTestType(Exception): class ChipToolTestType(str, Enum): CHIP_TOOL = "chip-tool" CHIP_APP = "chip-app" - PTYHON_TEST = "python-test" + PYTHON_TEST = "python-test" class ChipTool(metaclass=Singleton): @@ -266,7 +266,7 @@ async def start_chip_server( elif test_type == ChipToolTestType.CHIP_APP: prefix = CHIP_APP_EXE command = ["--interactive", "--port 9002"] - elif test_type == ChipToolTestType.PTYHON_TEST: + elif test_type == ChipToolTestType.PYTHON_TEST: pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") @@ -595,8 +595,8 @@ async def run_test( adapter = ChipToolAdapter.Adapter(parser_config.definitions) elif test_type == ChipToolTestType.CHIP_APP: adapter = ChipAppAdapter.Adapter(parser_config.definitions) - elif test_type == ChipToolTestType.PTYHON_TEST: - # TODO - run_test for python_test must be updated + elif test_type == ChipToolTestType.PYTHON_TEST: + # TODO - run_test() for python_test to be implemented pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 612e533e..61791c1b 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from .support.yaml_tests import sdk_collection from .support.python_testing import sdk_python_collection +from .support.yaml_tests import sdk_collection diff --git a/test_collections/sdk_tests/support/python_testing/models/__init__.py b/test_collections/sdk_tests/support/python_testing/models/__init__.py index 8b47fb7f..d15bf56b 100644 --- a/test_collections/sdk_tests/support/python_testing/models/__init__.py +++ b/test_collections/sdk_tests/support/python_testing/models/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. # from .test_case import PythonTestCase -from .test_suite import SuiteType, PythonTestSuite +from .test_suite import PythonTestSuite, SuiteType diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py index f8fb31a0..a7ec4ca4 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py @@ -19,6 +19,7 @@ VERSION_FILE_FILENAME = ".version" VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") + class PythonTestFolder: """Representing a folder with Test Python files. @@ -31,7 +32,7 @@ def __init__(self, path: Path, filename_pattern: str = "*") -> None: self.version = self.__version() def __version(self) -> str: - """Read version string from .version file in + """Read version string from .version file in /app/backend/test_collections/sdk_tests/sdk_checkout path.""" version_file_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index d3246fd3..4ca11a15 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -26,8 +26,6 @@ class PythonTestType(Enum): AUTOMATED = 0 - SEMI_AUTOMATED = 1 - SIMULATED = 2 class PythonTestStep(BaseModel): diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 9b3ea440..25d6445c 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -13,139 +13,82 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from pathlib import Path -from typing import List +# type: ignore +# Ignore mypy type check for this file import ast -from loguru import logger -from pydantic import ValidationError +from pathlib import Path +from typing import List, Tuple + +from .python_test_models import PythonTest, PythonTestStep, PythonTestType -from .python_test_models import PythonTest, PythonTestType, PythonTestStep +ARG_STEP_DESCRIPTION_INDEX = 1 +KEYWORD_IS_COMISSIONING_INDEX = 0 class PythonParserException(Exception): """Raised when an error occurs during the parser of python file.""" -def _test_type(test: PythonTest) -> PythonTestType: - """Determine the type of a test based on the parsed python. - - This is mainly determined by the number of disabled test steps. - - Args: - test (PythonTest): parsed python test - - Returns: - TestType: - - Manual: All steps disabled - - Semi-Automated: some steps are disabled - - Automated: no disabled steps - - Simulated: Tests where file name have "Simulated" - """ - if test.path is not None and "Simulated" in str(test.path): - return PythonTestType.SIMULATED - - steps = test.steps - - # if any step has a UserPrompt, categorize as semi-automated - if any(s.command == "UserPrompt" for s in steps): - return PythonTestType.SEMI_AUTOMATED - - # Otherwise Automated - return PythonTestType.AUTOMATED - - def parse_python_test(path: Path) -> PythonTest: """Parse a single Python test file into PythonTest model. This will also annotate parsed python test with it's path and test type. """ python_steps: list[PythonTestStep] = [] - tc_pics = [] - tc_config = {} + # Currently PICS and config is not configured in Python Testing + tc_pics: list = [] + tc_config: dict = {} + tc_desc, python_steps = __extract_tcs_info(path) - if "ACE_1_3" in str(path): - python_steps = tc_steps(path) - tc_desc = tc_description(path) - # with open(str(path)) as python_file: - # parsed = ast.parse(python_file.read()) - - # functions = [n for n in parsed.body if isinstance(n, ast.FunctionDef)] - # classes = [n for n in parsed.body if isinstance(n, ast.ClassDef)] - - # print(functions) - # print(classes) - - # for function in functions: - # result.append(show_info(function)) - - # for step in steps: - # python_steps = tc_steps(path) - # tc_desc = tc_description(path) - + if not tc_desc or not python_steps: + # The file name from path + tc_name = path.name.split(".")[0] + raise PythonParserException( + f"Test Case {tc_name} does not have methods desc_{tc_name} or steps_{tc_name}" + ) - return PythonTest( - name=tc_desc, tests=python_steps, config=tc_config, PICS=tc_pics - ) + test = PythonTest(name=tc_desc, tests=python_steps, config=tc_config, PICS=tc_pics) + test.path = path + test.type = PythonTestType.AUTOMATED -def tc_description(path: Path) -> str: - with open(str(path)) as python_file: - parsed = ast.parse(python_file.read()) + return test - # functions = [n for n in parsed.body if isinstance(n, ast.FunctionDef)] - classes = [n for n in parsed.body if isinstance(n, ast.ClassDef)] - # for function in functions: - # print(function) - # result.append(show_info(function)) +def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: + with open(path, "r") as python_file: + parsed_python_file = ast.parse(python_file.read()) + classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] - tc_desc = "'" - python_steps:List[PythonTestStep] = [] + tc_desc: str = "" + python_steps: List[PythonTestStep] = [] for class_ in classes: - methods = [n for n in class_.body if isinstance(n, ast.FunctionDef)] + methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] for method in methods: if "desc_" in method.name: tc_desc = method.body[0].value.value elif "steps_" in method.name: - python_steps = retrieve_steps(method) - # desc_TC_ACE_1_3 -> method.body[0].value.value - # method.name - # print(method) - # result.append((class_.name + "." + show_info(method))) - # # desc_* - # method.body[0].value.value - # # steps_* - # method.body[0].value.elts[4].args[1].value - - # method.body[0].value.elts[0].keywords[0].arg - # method.body[0].value.elts[0].keywords[0].value.value - - # print(", ".join(result)) - - return "" - -def retrieve_steps(method: ast.FunctionDef)-> List[PythonTestStep]: - python_steps:List[PythonTestStep] = [] + python_steps = __retrieve_steps(method) + + return tc_desc, python_steps + + +def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: + python_steps: List[PythonTestStep] = [] for step in method.body[0].value.elts: - step_name = step.args[1].value - step_is_commissioning = False - if step.keywords and 'is_commissioning' in step.keywords[0].arg: - is_commissioning = step.keywords[0].value.value + step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value + arg_is_commissioning = False + if ( + step.keywords + and "is_commissioning" in step.keywords[KEYWORD_IS_COMISSIONING_INDEX].arg + ): + arg_is_commissioning = step.keywords[ + KEYWORD_IS_COMISSIONING_INDEX + ].value.value python_steps.append( - PythonTestStep( - label=step_name, is_commissioning=step_is_commissioning - ) + PythonTestStep(label=step_name, is_commissioning=arg_is_commissioning) ) - - return python_steps -# def tc_steps(path: Path) -> List[PythonTestStep]: -# python_steps:List[PythonTestStep] = [] -# python_steps.append( -# PythonTestStep( -# label="step.description", is_commissioning=False # step.is_commissioning -# ) -# ) -# return python_steps + return python_steps diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 3c9d6149..f30f558d 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -75,22 +75,18 @@ async def setup(self) -> None: @classmethod def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: """Dynamically declares a subclass based on the type of Python test.""" - case_class: Type[PythonTestCase] - if test.type == PythonTestType.SEMI_AUTOMATED: - case_class = PythonSemiAutomatedChipToolTestCase - elif test.type == PythonTestType.SIMULATED: - case_class = PythonSimulatedTestCase - else: # Automated - case_class = PythonChipToolTestCase + case_class: Type[PythonTestCase] = PythonChipToolTestCase - return case_class.__class_factory(test=test, python_test_version=python_test_version) + return case_class.__class_factory( + test=test, python_test_version=python_test_version + ) @classmethod def __class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: """Common class factory method for all subclasses of PythonTestCase.""" identifier = cls.__test_identifier(test.name) class_name = cls.__class_name(identifier) - title = cls.__title(identifier=identifier, test_python=test) + title = identifier return type( class_name, @@ -124,31 +120,6 @@ def __class_name(identifier: str) -> str: """Replace all non-alphanumeric characters with _ to make valid class name.""" return re.sub("[^0-9a-zA-Z]+", "_", identifier) - @staticmethod - def __has_steps_disabled(test_python: PythonTest) -> bool: - """If some but not all steps are disabled, return true. False otherwise.""" - len_disabled_steps = len([s for s in test_python.steps if not s.disabled]) - - if len_disabled_steps == 0: - return False - else: - return len_disabled_steps < len(test_python.steps) - - @classmethod - def __title(cls, identifier: str, test_python: PythonTest) -> str: - """Annotate Title with Semi-automated and Steps Disabled tests in the test - title. - """ - title = identifier - - if test_python.type == PythonTestType.SEMI_AUTOMATED: - title += " (Semi-automated)" - - if cls.__has_steps_disabled(test_python): - title += " (Steps Disabled)" - - return title - def _append_automated_test_step(self, python_test_step: PythonTestStep) -> None: """ Disabled steps are ignored. @@ -173,28 +144,11 @@ def _append_automated_test_step(self, python_test_step: PythonTestStep) -> None: class PythonChipToolTestCase(PythonTestCase, ChipToolTest): - """Automated test cases using chip-tool.""" - - test_type = ChipToolTestType.CHIP_TOOL - - def create_test_steps(self) -> None: - self.test_steps = [TestStep("Start chip-tool test")] - for step in self.python_test.steps: - self._append_automated_test_step(step) - - -class PythonSemiAutomatedChipToolTestCase(PythonChipToolTestCase, ChipToolManualPromptTest): - """Semi-Automated test cases, need special step for users to attach logs - for manual steps, so inheriting from ChipToolManualPromptTest. - """ - - -class PythonSimulatedTestCase(PythonTestCase, ChipToolTest): - """Simulated test cases using chip-app""" + """Automated Python test cases.""" - test_type = ChipToolTestType.CHIP_APP + test_type = ChipToolTestType.PYTHON_TEST def create_test_steps(self) -> None: - self.test_steps = [TestStep("Start chip-app test")] + self.test_steps = [TestStep("Start Python test")] for step in self.python_test.steps: self._append_automated_test_step(step) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py index 1acd7363..8e30e22a 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -21,10 +21,10 @@ TestSuiteDeclaration, ) -from .test_case import PythonTestCase -from .test_suite import SuiteType, PythonTestSuite from .python_test_folder import PythonTestFolder from .python_test_models import PythonTest, PythonTestType +from .test_case import PythonTestCase +from .test_suite import PythonTestSuite, SuiteType class PythonCollectionDeclaration(TestCollectionDeclaration): @@ -55,7 +55,9 @@ class PythonCaseDeclaration(TestCaseDeclaration): def __init__(self, test: PythonTest, python_test_version: str) -> None: super().__init__( - PythonTestCase.class_factory(test=test, python_test_version=python_test_version) + PythonTestCase.class_factory( + test=test, python_test_version=python_test_version + ) ) @property diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index ff096fac..8ababa0b 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -27,8 +27,7 @@ class PythonTestSuiteFactoryError(Exception): class SuiteType(Enum): - SIMULATED = 1 - AUTOMATED = 2 + AUTOMATED = 1 # Custom Type variable used to annotate the factory methods of classmethod. @@ -56,12 +55,12 @@ def class_factory( """Dynamically declares a subclass based on the type of test suite.""" suite_class = PythonTestSuite - if suite_type == SuiteType.SIMULATED: - suite_class = SimulatedPythonTestSuite - elif suite_type == SuiteType.AUTOMATED: + if suite_type == SuiteType.AUTOMATED: suite_class = ChipToolPythonTestSuite - return suite_class.__class_factory(name=name, python_test_version=python_test_version) + return suite_class.__class_factory( + name=name, python_test_version=python_test_version + ) @classmethod def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: @@ -84,13 +83,9 @@ def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: class ChipToolPythonTestSuite(PythonTestSuite, ChipToolSuite): - test_type = ChipToolTestType.CHIP_TOOL + test_type = ChipToolTestType.PYTHON_TEST async def setup(self) -> None: """Due top multi inheritance, we need to call setup on both super classes.""" await PythonTestSuite.setup(self) await ChipToolSuite.setup(self) - - -class SimulatedPythonTestSuite(ChipToolPythonTestSuite): - test_type = ChipToolTestType.CHIP_APP diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py index a57dc024..3e91b76f 100644 --- a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -17,15 +17,14 @@ from loguru import logger +from .models.python_test_folder import PythonTestFolder +from .models.python_test_parser import PythonParserException, parse_python_test from .models.test_declarations import ( PythonCaseDeclaration, PythonCollectionDeclaration, PythonSuiteDeclaration, ) from .models.test_suite import SuiteType -from .models.python_test_folder import PythonTestFolder -from .models.python_test_models import PythonTestType -from .models.python_test_parser import PythonParserException, parse_python_test ### # This file hosts logic load and parse Python test-cases, located in @@ -37,10 +36,17 @@ # - Simulated using Chip-App1 ### -SDK_PYTHON_TEST_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk") -SDK_PYTHON_TEST_FOLDER = PythonTestFolder(path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*") +SDK_PYTHON_TEST_PATH = Path( + "/app/backend/test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk" +) +SDK_PYTHON_TEST_FOLDER = PythonTestFolder( + path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*" +) + -def _init_test_suites(python_test_version: str) -> dict[SuiteType, PythonSuiteDeclaration]: +def _init_test_suites( + python_test_version: str, +) -> dict[SuiteType, PythonSuiteDeclaration]: return { SuiteType.AUTOMATED: PythonSuiteDeclaration( name="Python Testing Suite", @@ -54,7 +60,9 @@ def _parse_python_test_to_test_case_declaration( python_test_path: Path, python_test_version: str ) -> PythonCaseDeclaration: python_test = parse_python_test(python_test_path) - return PythonCaseDeclaration(test=python_test, python_test_version=python_test_version) + return PythonCaseDeclaration( + test=python_test, python_test_version=python_test_version + ) def _parse_all_sdk_python_tests( @@ -70,17 +78,17 @@ def _parse_all_sdk_python_tests( for python_test_file in python_test_files: try: test_case = _parse_python_test_to_test_case_declaration( - python_test_path=python_test_file, python_test_version=python_test_version + python_test_path=python_test_file, + python_test_version=python_test_version, ) - if test_case.test_type == PythonTestType.SIMULATED: - suites[SuiteType.SIMULATED].add_test_case(test_case) - else: - suites[SuiteType.AUTOMATED].add_test_case(test_case) - except PythonParserException: + suites[SuiteType.AUTOMATED].add_test_case(test_case) + except PythonParserException as e: # If an exception was raised during parse process, the python file will be # ignored and the loop will continue with the next python file - logger.error(f"Error while parsing Python File: {python_test_file}") + logger.error( + f"Error while parsing Python File: {python_test_file} \nError:{e}" + ) return list(suites.values()) @@ -95,7 +103,9 @@ def sdk_python_test_collection( files = python_test_folder.python_file_paths() version = python_test_folder.version - suites = _parse_all_sdk_python_tests(python_test_files=files, python_test_version=version) + suites = _parse_all_sdk_python_tests( + python_test_files=files, python_test_version=version + ) for suite in suites: suite.sort_test_cases() From d54d375f0cf54fc022d073e8b11cf483cfa768ac Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 12:48:17 +0000 Subject: [PATCH 03/45] Changed init from sdk_tests --- test_collections/sdk_tests/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 61791c1b..6df5f6d8 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -13,5 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from .support.python_testing import sdk_python_collection -from .support.yaml_tests import sdk_collection +from .support.yaml_tests import sdk_collection, custom_collection From 19852304b53edac95fa63c1f8c5089285e85b036 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 13:15:16 +0000 Subject: [PATCH 04/45] Minor changes --- test_collections/sdk_tests/__init__.py | 1 + .../support/python_testing/models/test_case.py | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 6df5f6d8..39754097 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -14,3 +14,4 @@ # limitations under the License. # from .support.yaml_tests import sdk_collection, custom_collection +from .support.python_testing import sdk_python_collection \ No newline at end of file diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index f30f558d..a45fd75d 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -127,19 +127,8 @@ def _append_automated_test_step(self, python_test_step: PythonTestStep) -> None: UserPrompt are special cases that will prompt test operator for input. """ - if python_test_step.disabled: - test_engine_logger.info( - f"{self.public_id()}: skipping disabled step: {python_test_step.label}" - ) - return step = TestStep(python_test_step.label) - if python_test_step.command == "UserPrompt": - step = ManualVerificationTestStep( - name=python_test_step.label, - verification=python_test_step.verification, - ) - self.test_steps.append(step) From 91fdbf976726d085cb7af453d8086b5fed4e557c Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 14:01:32 +0000 Subject: [PATCH 05/45] Disable support to list python test cases --- test_collections/sdk_tests/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 39754097..93c0e680 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -14,4 +14,6 @@ # limitations under the License. # from .support.yaml_tests import sdk_collection, custom_collection -from .support.python_testing import sdk_python_collection \ No newline at end of file +# TODO The Python test support implementation is ongoing. +# Uncomment line below when the implementation is done. +#from .support.python_testing import sdk_python_collection \ No newline at end of file From df0e12984eb01636a221cef0b873b0b904121032 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 14:08:06 +0000 Subject: [PATCH 06/45] Reverted minor changes --- app/core/config.py | 2 +- .../python_tests/test_python_test_suite.py | 111 ++++++++++++++++++ .../yaml_tests/test_sdk_yaml_collection.py | 1 - app/tests/yaml_tests/test_yaml_folder.py | 8 +- .../support/yaml_tests/sdk_yaml_tests.py | 5 +- 5 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 app/tests/python_tests/test_python_test_suite.py diff --git a/app/core/config.py b/app/core/config.py index c6dd5ee5..ce41af1a 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -90,7 +90,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: # Logging LOGGING_PATH: str = "./logs" LOGGING_FILENAME: str = "{time:YYYY-MM-DD}.log" - LOGGING_LEVEL: str = "info" + LOGGING_LEVEL: str = "debug" LOGGING_ROTATION: str = "20 days" LOGGING_RETENTION: str = "1 months" LOGGING_FORMAT: str = ( diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py new file mode 100644 index 00000000..cd45323e --- /dev/null +++ b/app/tests/python_tests/test_python_test_suite.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Type +from unittest import mock + +import pytest + +from app.chip_tool.chip_tool import ChipToolTestType +from app.models.test_suite_execution import TestSuiteExecution +from app.test_engine.logger import test_engine_logger +from test_collections.sdk_tests.support.python_testing.models.test_suite import ( + ChipToolPythonTestSuite, + SuiteType, + PythonTestSuite, +) + + +def test_python_suite_class_factory_name() -> None: + """Test that test suite name is set.""" + name = "AnotherTestSuite" + + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=SuiteType.AUTOMATED, name=name, python_test_version="version" + ) + + assert suite_class.__name__ == name + assert suite_class.public_id() == name + assert suite_class.metadata["title"] == name + assert suite_class.metadata["description"] == name + + +def test_python_test_suite_python_version() -> None: + """Test that test suite python version is set correctly in class factory.""" + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=SuiteType.AUTOMATED, name="SomeSuite", python_test_version=python_test_version + ) + + assert suite_class.python_test_version == python_test_version + + +def test_automated_suite_subclass() -> None: + """Test that for suite type automated class factory creates a subclass of + ChipToolPythonTestSuite, and that test_type is set to CHIP_TOOL""" + type = SuiteType.AUTOMATED + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version="some_version" + ) + assert issubclass(suite_class, ChipToolPythonTestSuite) + assert suite_class.test_type == ChipToolTestType.PYTHON_TEST + + +@pytest.mark.asyncio +async def test_suite_setup_log_python_version() -> None: + """Test that test suite python version is logged to test engine logger in setup.""" + for type in list(SuiteType): + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version=python_test_version + ) + + suite_instance = suite_class(TestSuiteExecution()) + + # We're patching ChipToolSuite.setup to avoid starting chip-tool + with mock.patch.object( + target=test_engine_logger, attribute="info" + ) as logger_info, mock.patch( + "app.chip_tool.test_suite.ChipToolSuite.setup" + ) as _: + await suite_instance.setup() + logger_info.assert_called() + logger_info.assert_any_call(f"Python Test Version: {python_test_version}") + + +@pytest.mark.asyncio +async def test_chip_tool_suite_setup() -> None: + """Test that both PythonTestSuite.setup and ChipToolSuite.setup are called when + PythonChipToolsSuite.setup is called. We do this as PythonChipToolsSuite inherits from + both PythonTestSuite and ChipToolSuite.""" + + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=SuiteType.AUTOMATED, name="SomeSuite", python_test_version="Some version" + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_suite.PythonTestSuite.setup" + ) as python_suite_setup, mock.patch( + "app.chip_tool.test_suite.ChipToolSuite.setup" + ) as chip_tool_suite_setup: + await suite_instance.setup() + python_suite_setup.assert_called_once() + chip_tool_suite_setup.assert_called_once() diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index db81d4cc..d5891120 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -34,7 +34,6 @@ VERSION_FILE_FILENAME = ".version" VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") - @pytest.fixture def yaml_collection() -> YamlCollectionDeclaration: test_sdk_yaml_path = Path(__file__).parent / "test_yamls" diff --git a/app/tests/yaml_tests/test_yaml_folder.py b/app/tests/yaml_tests/test_yaml_folder.py index 65a5012e..81f77273 100644 --- a/app/tests/yaml_tests/test_yaml_folder.py +++ b/app/tests/yaml_tests/test_yaml_folder.py @@ -31,7 +31,9 @@ def test_yaml_folder_version() -> None: with mock.patch( "test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder.open", new=mock.mock_open(read_data=version_file_content), - ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: + ), mock.patch.object( + target=Path, attribute="exists", return_value=True + ) as _: yaml_folder = YamlTestFolder(test_yaml_path) assert yaml_folder.version == version_file_content @@ -39,7 +41,9 @@ def test_yaml_folder_version() -> None: def test_yaml_folder_version_missing() -> None: expected_version = "Unknown" - with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: + with mock.patch.object( + target=Path, attribute="exists", return_value=False + ) as _: yaml_folder = YamlTestFolder(test_yaml_path) assert yaml_folder.version == expected_version diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index ebc35e8d..517977ce 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -31,9 +31,8 @@ ### # This file hosts logic load and parse YAML test-cases, located in -# `test_collections/sdk_tests/sdk_checkout/yaml_tests/yaml/sdk`. -# The `sdk` sub-folder here is automatically maintained using the -# `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. +# `test_collections/yaml_tests/yaml/sdk`. The `sdk` sub-folder here is automatically +# maintained using the `scripts/fetch_sdk_yaml_tests_and_runner.sh` script. # # The YAML Tests are organized into 3 Test Suites: # - Automated and Semi-Automated using Chip-Tool From fe24421ef3d335512a24cecfbe85d36a8bbf6613 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 16:56:38 +0000 Subject: [PATCH 07/45] Added Unit tests --- app/tests/python_tests/test_python_folder.py | 59 ++++ app/tests/python_tests/test_python_parser.py | 87 ++++++ .../python_tests/test_python_script/.version | 1 + .../UnitTest_TC_ACL_1_1_Automated.py | 51 ++++ .../python_tests/test_python_test_case.py | 259 ++++++++++++++++++ .../test_python_test_declarations.py | 55 ++++ .../test_sdk_python_collection.py | 74 +++++ .../models/python_test_models.py | 2 +- .../models/python_test_parser.py | 15 +- .../python_testing/models/test_case.py | 25 +- .../python_testing/models/test_suite.py | 5 +- .../python_testing/sdk_python_test_tests.py | 20 +- 12 files changed, 609 insertions(+), 44 deletions(-) create mode 100644 app/tests/python_tests/test_python_folder.py create mode 100644 app/tests/python_tests/test_python_parser.py create mode 100644 app/tests/python_tests/test_python_script/.version create mode 100644 app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py create mode 100644 app/tests/python_tests/test_python_test_case.py create mode 100644 app/tests/python_tests/test_python_test_declarations.py create mode 100644 app/tests/python_tests/test_sdk_python_collection.py diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py new file mode 100644 index 00000000..69e2bade --- /dev/null +++ b/app/tests/python_tests/test_python_folder.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.python_test_folder import ( + PythonTestFolder, +) + +test_python_path = Path("/test/python") + + +def test_python_folder_version() -> None: + version_file_content = "python_test_version" + + # We mock open to read version_file_content and Path exists to ignore that we're + # testing with a fake path + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_folder.open", + new=mock.mock_open(read_data=version_file_content), + ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: + python_test_folder = PythonTestFolder(test_python_path) + + assert python_test_folder.version == version_file_content + + +def test_python_folder_version_missing() -> None: + expected_version = "Unknown" + with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: + python_folder = PythonTestFolder(test_python_path) + assert python_folder.version == expected_version + + +def test_python_folder_filename_pattern() -> None: + """Test PythonTestFolder will search for files with filename pattern.""" + with mock.patch.object(target=Path, attribute="glob") as path_glob: + # Default file_name_patter: * + python_folder = PythonTestFolder(test_python_path) + _ = python_folder.python_file_paths() + path_glob.assert_called_once_with("*.py") + + path_glob.reset_mock() + pattern = "TC_*" + python_test_folder = PythonTestFolder(test_python_path, filename_pattern=pattern) + _ = python_test_folder.python_file_paths() + path_glob.assert_called_once_with(f"{pattern}.py") diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py new file mode 100644 index 00000000..a85c8e36 --- /dev/null +++ b/app/tests/python_tests/test_python_parser.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.python_test_parser import ( + PythonParserException, + parse_python_test, +) + +sample_invalid_python_file_content = """ +class TC_Sample(MatterBaseTest): + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + @async_test_body + async def test_steps_TC_Sample(self): + print("Test execution") + +""" + +sample_python_file_content = """ +class TC_Sample(MatterBaseTest): + + def desc_TC_Sample(self) -> str: + return "Sample TC Description" + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + @async_test_body + async def test_steps_TC_Sample(self): + print("Test execution") + +""" + + +def test_python_file_parser_throws_pythonparserexception() -> None: + file_path = Path("/test/file.py") + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_parser.open", + new=mock.mock_open(read_data=sample_invalid_python_file_content), + ): + try: + parse_python_test(file_path) + except PythonParserException as e: + assert "Test Case file does not have methods desc_file or steps_file" == str(e) + + +def test_python_file_parser() -> None: + file_path = Path("/test/file.py") + + # We mock builtin `open` method to read sample python file content, + # to avoid having to load a real file. + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_parser.open", + new=mock.mock_open(read_data=sample_python_file_content), + ) as file_open: + test = parse_python_test(file_path) + + file_open.assert_called_once_with(file_path, "r") + assert test.path == file_path diff --git a/app/tests/python_tests/test_python_script/.version b/app/tests/python_tests/test_python_script/.version new file mode 100644 index 00000000..3e269350 --- /dev/null +++ b/app/tests/python_tests/test_python_script/.version @@ -0,0 +1 @@ +unit-test-python-version \ No newline at end of file diff --git a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py new file mode 100644 index 00000000..261a6bfe --- /dev/null +++ b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py @@ -0,0 +1,51 @@ + + # + # Copyright (c) 2023 Project CHIP Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# type: ignore +# Ignore mypy type check for this file + +""" +This is just a sample test case that should come from SDK. +It should not compile or run. +""" +class TC_Sample(MatterBaseTest): + + def desc_TC_Sample(self) -> str: + return "Sample TC Description" + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + @async_test_body + async def test_steps_TC_Sample(self): + print("Test execution") \ No newline at end of file diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py new file mode 100644 index 00000000..addedd91 --- /dev/null +++ b/app/tests/python_tests/test_python_test_case.py @@ -0,0 +1,259 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from typing import Any, Optional, Type +from unittest import mock + +import pytest + +from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.test_case import TestError +from app.models.test_case_execution import TestCaseExecution +from app.test_engine.logger import test_engine_logger +from test_collections.sdk_tests.support.python_testing.models import PythonTestCase +from test_collections.sdk_tests.support.python_testing.models.test_case import ( + PythonChipToolTestCase, +) +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, + PythonTestStep, + PythonTestType, +) + + +def python_test_instance( + name: str = "Test Python", + PICS: set[str] = {"PICS.A", "PICS.B"}, + config: dict[str, Any] = { + "param1": "value1", + "param2": {"type": "config_type", "defaultValue": "value2"}, + }, + steps: list[PythonTestStep] = [], + type: PythonTestType = PythonTestType.AUTOMATED, + path: Optional[Path] = None, +) -> PythonTest: + return PythonTest( + name=name, + PICS=PICS, + config=config, + steps=steps, + type=type, + path=path, + ) + + +def test_python_test_name() -> None: + """Test that test name is set as description in metadata.""" + name = "Another Test Name" + test = python_test_instance(name=name) + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.metadata["description"] == name + + +def test_python_test_python_version() -> None: + """Test that test case python version is set correctly in class factory.""" + test = python_test_instance() + python_test_version = "best_version" + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version=python_test_version + ) + assert case_class.python_test_version == python_test_version + + +def test_python_test_python() -> None: + """Test that test python_test property is as expected in subclass of PythonTestCase.""" + test = python_test_instance() + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.python_test is test + + +def test_python_test_case_class_pics() -> None: + """Test that the PICS of the python test is available in the class method PICS on + TestCase.""" + test_PICS = set(["PICS.D", "PICS.C"]) + test = python_test_instance(PICS=test_PICS) + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.pics() == test_PICS + + +def test_python_test_case_class_default_test_parameters() -> None: + """Test that the default_test_parameters of the python test is available in the class + method default_test_parameters on TestCase. + + Also parameters with type in Python test should be flattened and type dropped.""" + + test_input_config = { + "param1": "value1", + "param2": {"type": "config_type", "defaultValue": "value2"}, + } + + test = python_test_instance(config=test_input_config) + expected_default_test_parameters = {"param1": "value1", "param2": "value2"} + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.default_test_parameters() == expected_default_test_parameters + + +def test_automated_test_case_class_factory_subclass_mapping() -> None: + """Test Automated tests are created as a subclass of + PythonChipToolTestCase.""" + test = python_test_instance(type=PythonTestType.AUTOMATED) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert issubclass(case_class, PythonChipToolTestCase) + + +def test_class_factory_test_public_id() -> None: + """Test that class factory correctly finds identifier 'TC-XX-1.1' in python test name. + And set it as public_id in metadata""" + test_data = [ + {"name": "TC-AB-1.2", "public_id": "TC-AB-1.2"}, + {"name": "[TC-CD-3.4]", "public_id": "TC-CD-3.4"}, + {"name": "Test Name before [TC-EF-5.6]", "public_id": "TC-EF-5.6"}, + {"name": "[TC-GH-7.8] Test Name after", "public_id": "TC-GH-7.8"}, + {"name": "Before and [TC-IJ-9.0] after", "public_id": "TC-IJ-9.0"}, + {"name": "Before and TC-KL-10.11 after", "public_id": "TC-KL-10.11"}, + {"name": "TC-MORE-NAME-13.110", "public_id": "TC-MORE-NAME-13.110"}, + ] + for data in test_data: + test = python_test_instance(name=data["name"]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.metadata["public_id"] == data["public_id"] + + +def test_class_factory_test_class_name() -> None: + """Test that class factory correctly finds identifier 'TC-XX-1.1', convert it to + a safe class name, eg TC_XX_1_1""" + test_data = [ + {"name": "TC-AB-1.2", "class_name": "TC_AB_1_2"}, + {"name": "[TC-CD-3.4]", "class_name": "TC_CD_3_4"}, + {"name": "Test Name before [TC-EF-5.6]", "class_name": "TC_EF_5_6"}, + {"name": "[TC-GH-7.8] Test Name after", "class_name": "TC_GH_7_8"}, + {"name": "Before and [TC-IJ-9.0] after", "class_name": "TC_IJ_9_0"}, + {"name": "Before and TC-KL-10.11 after", "class_name": "TC_KL_10_11"}, + {"name": "TC-MORE-NAME-13.110", "class_name": "TC_MORE_NAME_13_110"}, + ] + for data in test_data: + test = python_test_instance(name=data["name"]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.__name__ == data["class_name"] + + +def test_test_type_for_automated_tests() -> None: + """Test that automated tests are set to use chip-tool""" + test = python_test_instance(type=PythonTestType.AUTOMATED) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert issubclass(case_class, PythonChipToolTestCase) + instance = case_class(TestCaseExecution()) + assert instance.test_type == ChipToolTestType.PYTHON_TEST + + +@pytest.mark.asyncio +async def test_python_version_logging() -> None: + """Test that all Python tests will log Python test version to test_engine_logger. + + Note that since `chip-tool` is not setup, we except the TestError raised. + """ + for type in list(PythonTestType): + test = python_test_instance(type=type) + test_python_version = "PythonVersionTest" + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version=test_python_version + ) + instance = case_class(TestCaseExecution()) + + with mock.patch.object( + target=test_engine_logger, attribute="info" + ) as logger_info: + try: + await instance.setup() + except TestError: + pass + logger_info.assert_called() + logger_info.assert_any_call(f"Python Test Version: {test_python_version}") + + +def test_default_first_steps_for_python_chip_tool_test_case() -> None: + test = python_test_instance(type=PythonTestType.AUTOMATED, steps=[]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + assert len(instance.test_steps) == 1 + assert instance.test_steps[0].name == "Start Python test" + + +def test_normal_steps_for_non_manual_tests() -> None: + """Test that non-manual tests include enabled steps.""" + for type in list(PythonTestType): + test_step = PythonTestStep(label="Step1") + test = python_test_instance(type=type, steps=[test_step]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + # Assert normal step is present + assert len(instance.test_steps) >= 1 + assert any(s.name == test_step.label for s in instance.test_steps) + + +def test_multiple_steps_for_non_manual() -> None: + """Test that non-manual tests multiple enabled steps are all included.""" + for type in list(PythonTestType): + test_step = PythonTestStep(label="StepN") + no_steps = 5 + test = python_test_instance(type=type, steps=([test_step] * no_steps)) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + + # Assert all steps from python test are added + assert len(instance.test_steps) >= no_steps + steps_from_python = [s for s in instance.test_steps if s.name == test_step.label] + assert len(steps_from_python) == no_steps + + +@pytest.mark.asyncio +async def test_setup_super_error_handling() -> None: + # ignore requirement to create_tests on init + with mock.patch("app.test_engine.models.test_case.TestCase.create_test_steps") as _: + test = PythonTestCase(TestCaseExecution()) + test.python_test_version = "some version" + # Assert this doesn't raise an exception + await test.setup() diff --git a/app/tests/python_tests/test_python_test_declarations.py b/app/tests/python_tests/test_python_test_declarations.py new file mode 100644 index 00000000..7d70b603 --- /dev/null +++ b/app/tests/python_tests/test_python_test_declarations.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( + PythonCaseDeclaration, + PythonSuiteDeclaration, +) +from test_collections.sdk_tests.support.python_testing.models.test_suite import SuiteType +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, +) + + +def test_python_suite_declaration() -> None: + name = "TestName" + type = SuiteType.AUTOMATED + version = "SomeVersionStr" + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_suite.PythonTestSuite.class_factory" + ) as class_factory, mock.patch( + "app.test_engine.models.test_declarations.TestSuiteDeclaration.__init__" + ) as declaration_init: + PythonSuiteDeclaration(name=name, suite_type=type, version=version) + class_factory.assert_called_once_with( + name=name, suite_type=type, python_test_version=version + ) + declaration_init.assert_called_once() + + +def test_python_case_declaration() -> None: + test = PythonTest(name="TestTest", config={}, steps=[]) + version = "SomeVersionStr" + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.class_factory" + ) as class_factory, mock.patch( + "app.test_engine.models.test_declarations.TestCaseDeclaration.__init__" + ) as declaration_init: + PythonCaseDeclaration(test=test, python_test_version=version) + class_factory.assert_called_once_with(test=test, python_test_version=version) + declaration_init.assert_called_once() diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py new file mode 100644 index 00000000..367a3bf9 --- /dev/null +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +import pytest + +from test_collections.sdk_tests.support.python_testing.models.python_test_folder import ( + PythonTestFolder, +) +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTestType, +) +from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( + PythonCaseDeclaration, + PythonCollectionDeclaration, +) +from test_collections.sdk_tests.support.python_testing.sdk_python_test_tests import ( + sdk_python_test_collection, +) + +VERSION_FILE_FILENAME = ".version" +VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") + + +@pytest.fixture +def python_test_collection() -> PythonCollectionDeclaration: + test_sdk_python_path = Path(__file__).parent / "test_python_script" + folder = PythonTestFolder( + path=test_sdk_python_path, filename_pattern="UnitTest_TC_*" + ) + return sdk_python_test_collection(folder) + + +def test_sdk_python_test_collection( + python_test_collection: PythonCollectionDeclaration, +) -> None: + assert python_test_collection.name == "SDK Python Tests" + assert len(python_test_collection.test_suites.keys()) == 1 + + # test version number + test_sdk_python_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME + with open(test_sdk_python_version_path, "r") as version_file: + assert ( + python_test_collection.python_test_version == version_file.read().rstrip() + ) + + +def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> None: + expected_automated_test_cases = 1 + + # Assert automated tests cases + assert "Python Testing Suite" in python_test_collection.test_suites.keys() + automated_suite = python_test_collection.test_suites["Python Testing Suite"] + assert len(automated_suite.test_cases) == expected_automated_test_cases + + type_count = dict.fromkeys(PythonTestType, 0) + for test_case in automated_suite.test_cases.values(): + assert isinstance(test_case, PythonCaseDeclaration) + type_count[test_case.test_type] += 1 + + assert type_count[PythonTestType.AUTOMATED] == expected_automated_test_cases diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index 4ca11a15..e639f7d1 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -41,6 +41,6 @@ class PythonTest(BaseModel): name: str PICS: set[str] = set() config: dict[str, Any] - steps: list[PythonTestStep] = Field(alias="tests") + steps: list[PythonTestStep] type: PythonTestType = PythonTestType.AUTOMATED path: Optional[Path] diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 25d6445c..4f22227f 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -34,21 +34,21 @@ def parse_python_test(path: Path) -> PythonTest: This will also annotate parsed python test with it's path and test type. """ - python_steps: list[PythonTestStep] = [] + tc_steps: list[PythonTestStep] = [] # Currently PICS and config is not configured in Python Testing tc_pics: list = [] tc_config: dict = {} - tc_desc, python_steps = __extract_tcs_info(path) + tc_desc, tc_steps = __extract_tcs_info(path) - if not tc_desc or not python_steps: + if not tc_desc or not tc_steps: # The file name from path tc_name = path.name.split(".")[0] raise PythonParserException( f"Test Case {tc_name} does not have methods desc_{tc_name} or steps_{tc_name}" ) - test = PythonTest(name=tc_desc, tests=python_steps, config=tc_config, PICS=tc_pics) + test = PythonTest(name=tc_desc, steps=tc_steps, config=tc_config, PICS=tc_pics) test.path = path test.type = PythonTestType.AUTOMATED @@ -60,8 +60,9 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: parsed_python_file = ast.parse(python_file.read()) classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] + # Get TC description and TC steps from python test file tc_desc: str = "" - python_steps: List[PythonTestStep] = [] + tc_steps: List[PythonTestStep] = [] for class_ in classes: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] @@ -69,9 +70,9 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: if "desc_" in method.name: tc_desc = method.body[0].value.value elif "steps_" in method.name: - python_steps = __retrieve_steps(method) + tc_steps = __retrieve_steps(method) - return tc_desc, python_steps + return tc_desc, tc_steps def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index a45fd75d..a1da6185 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -17,16 +17,11 @@ from typing import Any, Type, TypeVar from app.chip_tool.chip_tool import ChipToolTestType -from app.chip_tool.test_case import ChipToolManualPromptTest, ChipToolTest +from app.chip_tool.test_case import ChipToolTest from app.test_engine.logger import test_engine_logger -from app.test_engine.models import ( - ManualTestCase, - ManualVerificationTestStep, - TestCase, - TestStep, -) +from app.test_engine.models import TestCase, TestStep -from .python_test_models import PythonTest, PythonTestStep, PythonTestType +from .python_test_models import PythonTest # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") @@ -120,17 +115,6 @@ def __class_name(identifier: str) -> str: """Replace all non-alphanumeric characters with _ to make valid class name.""" return re.sub("[^0-9a-zA-Z]+", "_", identifier) - def _append_automated_test_step(self, python_test_step: PythonTestStep) -> None: - """ - Disabled steps are ignored. - (Such tests will be marked as 'Steps Disabled' elsewhere) - - UserPrompt are special cases that will prompt test operator for input. - """ - - step = TestStep(python_test_step.label) - self.test_steps.append(step) - class PythonChipToolTestCase(PythonTestCase, ChipToolTest): """Automated Python test cases.""" @@ -140,4 +124,5 @@ class PythonChipToolTestCase(PythonTestCase, ChipToolTest): def create_test_steps(self) -> None: self.test_steps = [TestStep("Start Python test")] for step in self.python_test.steps: - self._append_automated_test_step(step) + python_test_step = TestStep(step.label) + self.test_steps.append(python_test_step) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 8ababa0b..5b13e4a6 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -53,10 +53,7 @@ def class_factory( cls, suite_type: SuiteType, name: str, python_test_version: str ) -> Type[T]: """Dynamically declares a subclass based on the type of test suite.""" - suite_class = PythonTestSuite - - if suite_type == SuiteType.AUTOMATED: - suite_class = ChipToolPythonTestSuite + suite_class = ChipToolPythonTestSuite return suite_class.__class_factory( name=name, python_test_version=python_test_version diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py index 3e91b76f..c16d10d4 100644 --- a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -28,12 +28,12 @@ ### # This file hosts logic load and parse Python test-cases, located in -# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. The `sdk` sub-folder here is automatically -# maintained using the `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. +# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. +# The `sdk` sub-folder here is automatically maintained using the +# `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. # -# The Python Tests are organized into 2 Test Suites: -# - Automated and Semi-Automated using Chip-Tool -# - Simulated using Chip-App1 +# The Python Tests are organized into 1 Test Suite: +# - Automated ### SDK_PYTHON_TEST_PATH = Path( @@ -68,11 +68,7 @@ def _parse_python_test_to_test_case_declaration( def _parse_all_sdk_python_tests( python_test_files: list[Path], python_test_version: str ) -> list[PythonSuiteDeclaration]: - """Parse all python test files and organize them in the 3 test suites: - - Automated and Semi-Automated using Chip-Tool - - Simulated using Chip-App1 - - Manual - """ + """Parse all python test files and add them into Automated Suite""" suites = _init_test_suites(python_test_version) for python_test_file in python_test_files: @@ -85,7 +81,7 @@ def _parse_all_sdk_python_tests( suites[SuiteType.AUTOMATED].add_test_case(test_case) except PythonParserException as e: # If an exception was raised during parse process, the python file will be - # ignored and the loop will continue with the next python file + # ignored and the loop will continue with the next file logger.error( f"Error while parsing Python File: {python_test_file} \nError:{e}" ) @@ -96,7 +92,7 @@ def _parse_all_sdk_python_tests( def sdk_python_test_collection( python_test_folder: PythonTestFolder = SDK_PYTHON_TEST_FOLDER, ) -> PythonCollectionDeclaration: - """Declare a new collection of test suites with the 3 test suites.""" + """Declare a new collection of test suites.""" collection = PythonCollectionDeclaration( name="SDK Python Tests", folder=python_test_folder ) From e8c6ef74d29e498cc39d1941ba5fb342286846dd Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 17:32:50 +0000 Subject: [PATCH 08/45] Minor change --- .../support/python_testing/models/python_test_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 4f22227f..2cef81ee 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -23,6 +23,7 @@ ARG_STEP_DESCRIPTION_INDEX = 1 KEYWORD_IS_COMISSIONING_INDEX = 0 +BODY_INDEX = 0 class PythonParserException(Exception): @@ -68,7 +69,7 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] for method in methods: if "desc_" in method.name: - tc_desc = method.body[0].value.value + tc_desc = method.body[BODY_INDEX].value.value elif "steps_" in method.name: tc_steps = __retrieve_steps(method) @@ -77,7 +78,7 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: python_steps: List[PythonTestStep] = [] - for step in method.body[0].value.elts: + for step in method.body[BODY_INDEX].value.elts: step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value arg_is_commissioning = False if ( From b682e0d72edfd621e6f417d0ed0921ddc5c18b2a Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 21:37:59 +0000 Subject: [PATCH 09/45] Fix merge conflict --- test_collections/sdk_tests/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 80b756ac..328aa5be 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -13,11 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -<<<<<<< HEAD from .support.yaml_tests import sdk_collection, custom_collection # TODO The Python test support implementation is ongoing. # Uncomment line below when the implementation is done. #from .support.python_testing import sdk_python_collection -======= -from .support.yaml_tests import custom_collection, sdk_collection ->>>>>>> main From 1b8e180a01c0ad1bf1846d7f0933015615c09e6c Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 22:59:04 +0000 Subject: [PATCH 10/45] Fix CI issues --- app/tests/python_tests/test_python_folder.py | 4 ++- app/tests/python_tests/test_python_parser.py | 4 ++- .../UnitTest_TC_ACL_1_1_Automated.py | 34 +++++++++---------- .../python_tests/test_python_test_case.py | 12 ++++--- .../test_python_test_declarations.py | 8 +++-- .../python_tests/test_python_test_suite.py | 10 ++++-- .../test_sdk_python_collection.py | 22 ++++++------ .../yaml_tests/test_sdk_yaml_collection.py | 16 ++++----- test_collections/sdk_tests/__init__.py | 7 ++-- .../yaml_tests/models/yaml_test_folder.py | 1 - 10 files changed, 65 insertions(+), 53 deletions(-) diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py index 69e2bade..2912e8d3 100644 --- a/app/tests/python_tests/test_python_folder.py +++ b/app/tests/python_tests/test_python_folder.py @@ -54,6 +54,8 @@ def test_python_folder_filename_pattern() -> None: path_glob.reset_mock() pattern = "TC_*" - python_test_folder = PythonTestFolder(test_python_path, filename_pattern=pattern) + python_test_folder = PythonTestFolder( + test_python_path, filename_pattern=pattern + ) _ = python_test_folder.python_file_paths() path_glob.assert_called_once_with(f"{pattern}.py") diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py index a85c8e36..645b216f 100644 --- a/app/tests/python_tests/test_python_parser.py +++ b/app/tests/python_tests/test_python_parser.py @@ -69,7 +69,9 @@ def test_python_file_parser_throws_pythonparserexception() -> None: try: parse_python_test(file_path) except PythonParserException as e: - assert "Test Case file does not have methods desc_file or steps_file" == str(e) + assert ( + "Test Case file does not have methods desc_file or steps_file" == str(e) + ) def test_python_file_parser() -> None: diff --git a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py index 261a6bfe..00d5a8be 100644 --- a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py +++ b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py @@ -1,18 +1,17 @@ - - # - # Copyright (c) 2023 Project CHIP Authors - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Copyright (c) 2021 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,8 +32,9 @@ This is just a sample test case that should come from SDK. It should not compile or run. """ -class TC_Sample(MatterBaseTest): + +class TC_Sample(MatterBaseTest): def desc_TC_Sample(self) -> str: return "Sample TC Description" @@ -48,4 +48,4 @@ def steps_TC_Sample(self) -> list[TestStep]: @async_test_body async def test_steps_TC_Sample(self): - print("Test execution") \ No newline at end of file + print("Test execution") diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index addedd91..f8cb909d 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -24,14 +24,14 @@ from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger from test_collections.sdk_tests.support.python_testing.models import PythonTestCase -from test_collections.sdk_tests.support.python_testing.models.test_case import ( - PythonChipToolTestCase, -) from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( PythonTest, PythonTestStep, PythonTestType, ) +from test_collections.sdk_tests.support.python_testing.models.test_case import ( + PythonChipToolTestCase, +) def python_test_instance( @@ -48,7 +48,7 @@ def python_test_instance( return PythonTest( name=name, PICS=PICS, - config=config, + config=config, steps=steps, type=type, path=path, @@ -245,7 +245,9 @@ def test_multiple_steps_for_non_manual() -> None: # Assert all steps from python test are added assert len(instance.test_steps) >= no_steps - steps_from_python = [s for s in instance.test_steps if s.name == test_step.label] + steps_from_python = [ + s for s in instance.test_steps if s.name == test_step.label + ] assert len(steps_from_python) == no_steps diff --git a/app/tests/python_tests/test_python_test_declarations.py b/app/tests/python_tests/test_python_test_declarations.py index 7d70b603..12d34604 100644 --- a/app/tests/python_tests/test_python_test_declarations.py +++ b/app/tests/python_tests/test_python_test_declarations.py @@ -15,13 +15,15 @@ # from unittest import mock +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, +) from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( PythonCaseDeclaration, PythonSuiteDeclaration, ) -from test_collections.sdk_tests.support.python_testing.models.test_suite import SuiteType -from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( - PythonTest, +from test_collections.sdk_tests.support.python_testing.models.test_suite import ( + SuiteType, ) diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py index cd45323e..bc701610 100644 --- a/app/tests/python_tests/test_python_test_suite.py +++ b/app/tests/python_tests/test_python_test_suite.py @@ -23,8 +23,8 @@ from app.test_engine.logger import test_engine_logger from test_collections.sdk_tests.support.python_testing.models.test_suite import ( ChipToolPythonTestSuite, - SuiteType, PythonTestSuite, + SuiteType, ) @@ -48,7 +48,9 @@ def test_python_test_suite_python_version() -> None: python_test_version = "best_version" # Create a subclass of PythonTestSuite suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( - suite_type=SuiteType.AUTOMATED, name="SomeSuite", python_test_version=python_test_version + suite_type=SuiteType.AUTOMATED, + name="SomeSuite", + python_test_version=python_test_version, ) assert suite_class.python_test_version == python_test_version @@ -96,7 +98,9 @@ async def test_chip_tool_suite_setup() -> None: both PythonTestSuite and ChipToolSuite.""" suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( - suite_type=SuiteType.AUTOMATED, name="SomeSuite", python_test_version="Some version" + suite_type=SuiteType.AUTOMATED, + name="SomeSuite", + python_test_version="Some version", ) suite_instance = suite_class(TestSuiteExecution()) diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index 367a3bf9..2b695a04 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -44,18 +44,18 @@ def python_test_collection() -> PythonCollectionDeclaration: return sdk_python_test_collection(folder) -def test_sdk_python_test_collection( - python_test_collection: PythonCollectionDeclaration, -) -> None: - assert python_test_collection.name == "SDK Python Tests" - assert len(python_test_collection.test_suites.keys()) == 1 +# def test_sdk_python_test_collection( +# python_test_collection: PythonCollectionDeclaration, +# ) -> None: +# assert python_test_collection.name == "SDK Python Tests" +# assert len(python_test_collection.test_suites.keys()) == 1 - # test version number - test_sdk_python_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME - with open(test_sdk_python_version_path, "r") as version_file: - assert ( - python_test_collection.python_test_version == version_file.read().rstrip() - ) +# # test version number +# test_sdk_python_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME +# with open(test_sdk_python_version_path, "r") as version_file: +# assert ( +# python_test_collection.python_test_version == version_file.read().rstrip() +# ) def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> None: diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index db81d4cc..d1f1b91d 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -42,14 +42,14 @@ def yaml_collection() -> YamlCollectionDeclaration: return sdk_yaml_test_collection(folder) -def test_sdk_yaml_collection(yaml_collection: YamlCollectionDeclaration) -> None: - assert yaml_collection.name == "SDK YAML Tests" - assert len(yaml_collection.test_suites.keys()) == 3 - - # test version number - test_sdk_yaml_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME - with open(test_sdk_yaml_version_path, "r") as version_file: - assert yaml_collection.yaml_version == version_file.read().rstrip() +# def test_sdk_yaml_collection(yaml_collection: YamlCollectionDeclaration) -> None: +# assert yaml_collection.name == "SDK YAML Tests" +# assert len(yaml_collection.test_suites.keys()) == 3 + +# # test version number +# test_sdk_yaml_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME +# with open(test_sdk_yaml_version_path, "r") as version_file: +# assert yaml_collection.yaml_version == version_file.read().rstrip() def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 328aa5be..1460bd59 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from .support.yaml_tests import sdk_collection, custom_collection +from .support.yaml_tests import custom_collection, sdk_collection + # TODO The Python test support implementation is ongoing. -# Uncomment line below when the implementation is done. -#from .support.python_testing import sdk_python_collection +# Uncomment line below when the implementation is done. +# from .support.python_testing import sdk_python_collection diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py index b8362b43..a9aaa313 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py @@ -21,7 +21,6 @@ VERSION_FILE_FILENAME = ".version" - class YamlTestFolder: """Representing a folder with Test YAML files. From 6b1bc8631ff02b65e007341db2f55120f7afa828 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 8 Nov 2023 23:58:07 +0000 Subject: [PATCH 11/45] Fix mypy and flake violations --- app/tests/python_tests/test_python_folder.py | 5 ++++- app/tests/python_tests/test_python_parser.py | 8 ++++++-- .../test_python_script/UnitTest_TC_ACL_1_1_Automated.py | 4 +++- app/tests/python_tests/test_python_test_case.py | 2 ++ app/tests/python_tests/test_python_test_declarations.py | 2 ++ app/tests/python_tests/test_sdk_python_collection.py | 2 ++ .../support/python_testing/models/python_test_models.py | 2 +- .../support/python_testing/models/python_test_parser.py | 9 ++++----- .../sdk_tests/support/python_testing/models/test_case.py | 7 ++++--- 9 files changed, 28 insertions(+), 13 deletions(-) diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py index 2912e8d3..8b113460 100644 --- a/app/tests/python_tests/test_python_folder.py +++ b/app/tests/python_tests/test_python_folder.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# flake8: noqa +# Ignore flake8 check for this file from pathlib import Path from unittest import mock @@ -29,7 +31,8 @@ def test_python_folder_version() -> None: # We mock open to read version_file_content and Path exists to ignore that we're # testing with a fake path with mock.patch( - "test_collections.sdk_tests.support.python_testing.models.python_test_folder.open", + "test_collections.sdk_tests.support.python_testing.models." + "python_test_folder.open", new=mock.mock_open(read_data=version_file_content), ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: python_test_folder = PythonTestFolder(test_python_path) diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py index 645b216f..24167627 100644 --- a/app/tests/python_tests/test_python_parser.py +++ b/app/tests/python_tests/test_python_parser.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# flake8: noqa +# Ignore flake8 check for this file from pathlib import Path from unittest import mock @@ -63,7 +65,8 @@ def test_python_file_parser_throws_pythonparserexception() -> None: file_path = Path("/test/file.py") with mock.patch( - "test_collections.sdk_tests.support.python_testing.models.python_test_parser.open", + "test_collections.sdk_tests.support.python_testing.models.python_test_parser." + "open", new=mock.mock_open(read_data=sample_invalid_python_file_content), ): try: @@ -80,7 +83,8 @@ def test_python_file_parser() -> None: # We mock builtin `open` method to read sample python file content, # to avoid having to load a real file. with mock.patch( - "test_collections.sdk_tests.support.python_testing.models.python_test_parser.open", + "test_collections.sdk_tests.support.python_testing.models.python_test_parser." + "open", new=mock.mock_open(read_data=sample_python_file_content), ) as file_open: test = parse_python_test(file_path) diff --git a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py index 00d5a8be..fd8f1972 100644 --- a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py +++ b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py @@ -25,9 +25,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# # type: ignore # Ignore mypy type check for this file - +# flake8: noqa +# Ignore flake8 check for this file """ This is just a sample test case that should come from SDK. It should not compile or run. diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index f8cb909d..1cc88f4a 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# flake8: noqa +# Ignore flake8 check for this file from pathlib import Path from typing import Any, Optional, Type from unittest import mock diff --git a/app/tests/python_tests/test_python_test_declarations.py b/app/tests/python_tests/test_python_test_declarations.py index 12d34604..5c797a8b 100644 --- a/app/tests/python_tests/test_python_test_declarations.py +++ b/app/tests/python_tests/test_python_test_declarations.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# flake8: noqa +# Ignore flake8 check for this file from unittest import mock from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index 2b695a04..949c653e 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# flake8: noqa +# Ignore flake8 check for this file from pathlib import Path import pytest diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index e639f7d1..51a6860e 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel ### # This file declares Python test models that are used to parse the Python Test Cases. diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 2cef81ee..d1eca07e 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# type: ignore -# Ignore mypy type check for this file import ast from pathlib import Path from typing import List, Tuple @@ -46,7 +44,8 @@ def parse_python_test(path: Path) -> PythonTest: # The file name from path tc_name = path.name.split(".")[0] raise PythonParserException( - f"Test Case {tc_name} does not have methods desc_{tc_name} or steps_{tc_name}" + f"Test Case {tc_name} does not have methods desc_{tc_name} " + f"or steps_{tc_name}" ) test = PythonTest(name=tc_desc, steps=tc_steps, config=tc_config, PICS=tc_pics) @@ -69,7 +68,7 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] for method in methods: if "desc_" in method.name: - tc_desc = method.body[BODY_INDEX].value.value + tc_desc = method.body[BODY_INDEX].value.value # type: ignore elif "steps_" in method.name: tc_steps = __retrieve_steps(method) @@ -78,7 +77,7 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: python_steps: List[PythonTestStep] = [] - for step in method.body[BODY_INDEX].value.elts: + for step in method.body[BODY_INDEX].value.elts: # type: ignore step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value arg_is_commissioning = False if ( diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index a1da6185..5d3a6cb2 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -33,8 +33,8 @@ class PythonTestCase(TestCase): This class provides a class factory that will dynamically declare a new sub-class based on the test-type the Python test is expressing. - The PythonTest will be stored as a class property that will be used at run-time in all - instances of such subclass. + The PythonTest will be stored as a class property that will be used at run-time + in all instances of such subclass. """ python_test: PythonTest @@ -47,7 +47,8 @@ def pics(cls) -> set[str]: @classmethod def default_test_parameters(cls) -> dict[str, Any]: - """Python test config dict, sometimes have a nested dict with type and default value. + """Python test config dict, sometimes have a nested dict with type + and default value. Only defaultValue is used in this case. """ parameters = {} From ba061c23cb3b1586e96db695c8304c462371347a Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Thu, 9 Nov 2023 00:00:44 +0000 Subject: [PATCH 12/45] Fix mypy and flake violations --- .../support/python_testing/models/python_test_parser.py | 4 ++-- .../sdk_tests/support/python_testing/models/test_case.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index d1eca07e..557fd723 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -68,7 +68,7 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] for method in methods: if "desc_" in method.name: - tc_desc = method.body[BODY_INDEX].value.value # type: ignore + tc_desc = method.body[BODY_INDEX].value.value # type: ignore elif "steps_" in method.name: tc_steps = __retrieve_steps(method) @@ -77,7 +77,7 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: python_steps: List[PythonTestStep] = [] - for step in method.body[BODY_INDEX].value.elts: # type: ignore + for step in method.body[BODY_INDEX].value.elts: # type: ignore step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value arg_is_commissioning = False if ( diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 5d3a6cb2..7861636a 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -33,7 +33,7 @@ class PythonTestCase(TestCase): This class provides a class factory that will dynamically declare a new sub-class based on the test-type the Python test is expressing. - The PythonTest will be stored as a class property that will be used at run-time + The PythonTest will be stored as a class property that will be used at run-time in all instances of such subclass. """ @@ -47,7 +47,7 @@ def pics(cls) -> set[str]: @classmethod def default_test_parameters(cls) -> dict[str, Any]: - """Python test config dict, sometimes have a nested dict with type + """Python test config dict, sometimes have a nested dict with type and default value. Only defaultValue is used in this case. """ From 817f72adf76087bc08ce235216364235491cf47e Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Thu, 9 Nov 2023 00:06:11 +0000 Subject: [PATCH 13/45] Fix mypy and flake violations --- app/tests/python_tests/test_python_test_suite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py index bc701610..28ed6631 100644 --- a/app/tests/python_tests/test_python_test_suite.py +++ b/app/tests/python_tests/test_python_test_suite.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# flake8: noqa +# Ignore flake8 check for this file from typing import Type from unittest import mock From beb46b3ff6afdec40a00e5d164ef658a572d6522 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Thu, 9 Nov 2023 00:53:37 +0000 Subject: [PATCH 14/45] Fixed unit tests --- .../test_sdk_python_collection.py | 40 ++++++++++--------- .../yaml_tests/test_sdk_yaml_collection.py | 36 ++++++++++------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index 949c653e..ff6534fa 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -16,6 +16,7 @@ # flake8: noqa # Ignore flake8 check for this file from pathlib import Path +from unittest import mock import pytest @@ -33,31 +34,34 @@ sdk_python_test_collection, ) -VERSION_FILE_FILENAME = ".version" -VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") - @pytest.fixture def python_test_collection() -> PythonCollectionDeclaration: test_sdk_python_path = Path(__file__).parent / "test_python_script" - folder = PythonTestFolder( - path=test_sdk_python_path, filename_pattern="UnitTest_TC_*" - ) - return sdk_python_test_collection(folder) + with mock.patch.object(Path, "exists", return_value=True), mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_folder.open", + new=mock.mock_open(read_data="unit-test-python-version"), + ): + folder = PythonTestFolder( + path=test_sdk_python_path, filename_pattern="UnitTest_TC_*" + ) + return sdk_python_test_collection(folder) -# def test_sdk_python_test_collection( -# python_test_collection: PythonCollectionDeclaration, -# ) -> None: -# assert python_test_collection.name == "SDK Python Tests" -# assert len(python_test_collection.test_suites.keys()) == 1 +def test_sdk_python_test_collection( + python_test_collection: PythonCollectionDeclaration, +) -> None: + assert python_test_collection.name == "SDK Python Tests" + assert len(python_test_collection.test_suites.keys()) == 1 -# # test version number -# test_sdk_python_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME -# with open(test_sdk_python_version_path, "r") as version_file: -# assert ( -# python_test_collection.python_test_version == version_file.read().rstrip() -# ) + # test version number + test_sdk_python_version_path = ( + "/app/backend/app/tests/python_tests/test_python_script/.version" + ) + with open(test_sdk_python_version_path, "r") as version_file: + assert ( + python_test_collection.python_test_version == version_file.read().rstrip() + ) def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> None: diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index d1f1b91d..b2c892bd 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -14,6 +14,7 @@ # limitations under the License. # from pathlib import Path +from unittest import mock import pytest @@ -31,25 +32,30 @@ sdk_yaml_test_collection, ) -VERSION_FILE_FILENAME = ".version" -VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") - @pytest.fixture def yaml_collection() -> YamlCollectionDeclaration: test_sdk_yaml_path = Path(__file__).parent / "test_yamls" - folder = YamlTestFolder(path=test_sdk_yaml_path, filename_pattern="UnitTest_TC_*") - return sdk_yaml_test_collection(folder) - - -# def test_sdk_yaml_collection(yaml_collection: YamlCollectionDeclaration) -> None: -# assert yaml_collection.name == "SDK YAML Tests" -# assert len(yaml_collection.test_suites.keys()) == 3 - -# # test version number -# test_sdk_yaml_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME -# with open(test_sdk_yaml_version_path, "r") as version_file: -# assert yaml_collection.yaml_version == version_file.read().rstrip() + with mock.patch.object(Path, "exists", return_value=True), mock.patch( + "test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder.open", + new=mock.mock_open(read_data="unit-test-yaml-version"), + ): + folder = YamlTestFolder( + path=test_sdk_yaml_path, filename_pattern="UnitTest_TC_*" + ) + return sdk_yaml_test_collection(folder) + + +def test_sdk_yaml_collection( + yaml_collection: YamlCollectionDeclaration, +) -> None: + assert yaml_collection.name == "SDK YAML Tests" + assert len(yaml_collection.test_suites.keys()) == 3 + + # test version number + test_sdk_yaml_path = "/app/backend/app/tests/yaml_tests/test_yamls/.version" + with open(test_sdk_yaml_path, "r") as version_file: + assert yaml_collection.yaml_version == version_file.read().rstrip() def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: From 8da8227c0fd5fdb2190c09fc2d0262fb825e13d9 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Fri, 10 Nov 2023 17:31:13 +0000 Subject: [PATCH 15/45] Add iniial code for Python Test runner --- app/chip_tool/chip_tool.py | 1 + app/chip_tool/test_case.py | 2 +- app/tests/python_tests/test_python_folder.py | 2 +- .../python_tests/test_python_test_case.py | 14 +- .../yaml_tests/test_sdk_yaml_collection.py | 4 +- .../models/python_test_folder.py | 5 +- .../models/python_test_parser.py | 48 +++-- .../models/python_testing_hooks_proxy.py | 95 +++++++++ .../python_testing/models/test_case.py | 180 ++++++++++++++++-- .../python_testing/sdk_python_test_tests.py | 8 +- 10 files changed, 314 insertions(+), 45 deletions(-) create mode 100644 test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 8a96ff2e..ff88efb7 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -267,6 +267,7 @@ async def start_chip_server( prefix = CHIP_APP_EXE command = ["--interactive", "--port 9002"] elif test_type == ChipToolTestType.PYTHON_TEST: + # TODO - To be implemented pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") diff --git a/app/chip_tool/test_case.py b/app/chip_tool/test_case.py index d1881c11..db93970e 100644 --- a/app/chip_tool/test_case.py +++ b/app/chip_tool/test_case.py @@ -108,7 +108,7 @@ def test_start(self, filename: str, name: str, count: int) -> None: # since there is step execute outside runner context self.next_step() - def test_stop(self, duration: int) -> None: + def test_stop(self, exception: Exception, duration: int) -> None: self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py index 8b113460..db690688 100644 --- a/app/tests/python_tests/test_python_folder.py +++ b/app/tests/python_tests/test_python_folder.py @@ -50,7 +50,7 @@ def test_python_folder_version_missing() -> None: def test_python_folder_filename_pattern() -> None: """Test PythonTestFolder will search for files with filename pattern.""" with mock.patch.object(target=Path, attribute="glob") as path_glob: - # Default file_name_patter: * + # Default file_name_pattern: * python_folder = PythonTestFolder(test_python_path) _ = python_folder.python_file_paths() path_glob.assert_called_once_with("*.py") diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index 1cc88f4a..0a3cbb2f 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -210,17 +210,7 @@ async def test_python_version_logging() -> None: logger_info.assert_any_call(f"Python Test Version: {test_python_version}") -def test_default_first_steps_for_python_chip_tool_test_case() -> None: - test = python_test_instance(type=PythonTestType.AUTOMATED, steps=[]) - case_class: Type[PythonTestCase] = PythonTestCase.class_factory( - test=test, python_test_version="version" - ) - instance = case_class(TestCaseExecution()) - assert len(instance.test_steps) == 1 - assert instance.test_steps[0].name == "Start Python test" - - -def test_normal_steps_for_non_manual_tests() -> None: +def test_normal_steps_for_python_tests() -> None: """Test that non-manual tests include enabled steps.""" for type in list(PythonTestType): test_step = PythonTestStep(label="Step1") @@ -234,7 +224,7 @@ def test_normal_steps_for_non_manual_tests() -> None: assert any(s.name == test_step.label for s in instance.test_steps) -def test_multiple_steps_for_non_manual() -> None: +def test_multiple_steps_for_python_tests() -> None: """Test that non-manual tests multiple enabled steps are all included.""" for type in list(PythonTestType): test_step = PythonTestStep(label="StepN") diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index b2c892bd..8319bbc9 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -53,8 +53,8 @@ def test_sdk_yaml_collection( assert len(yaml_collection.test_suites.keys()) == 3 # test version number - test_sdk_yaml_path = "/app/backend/app/tests/yaml_tests/test_yamls/.version" - with open(test_sdk_yaml_path, "r") as version_file: + test_sdk_yaml_version_path = "/app/backend/app/tests/yaml_tests/test_yamls/.version" + with open(test_sdk_yaml_version_path, "r") as version_file: assert yaml_collection.yaml_version == version_file.read().rstrip() diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py index a7ec4ca4..92410538 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py @@ -15,9 +15,10 @@ # from pathlib import Path +from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH + UNKNOWN_version = "Unknown" VERSION_FILE_FILENAME = ".version" -VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") class PythonTestFolder: @@ -34,7 +35,7 @@ def __init__(self, path: Path, filename_pattern: str = "*") -> None: def __version(self) -> str: """Read version string from .version file in /app/backend/test_collections/sdk_tests/sdk_checkout path.""" - version_file_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME + version_file_path = SDK_CHECKOUT_PATH / VERSION_FILE_FILENAME if not version_file_path.exists(): return UNKNOWN_version diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 557fd723..e5f6d9b9 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -28,19 +28,33 @@ class PythonParserException(Exception): """Raised when an error occurs during the parser of python file.""" +class PythonTestInfo: + """This class stores all the information from a python test case that came from + python test script file.""" + + def __init__( + self, + desc: str, + pics: list, + config: dict, + steps: list[PythonTestStep], + type: PythonTestType, + ) -> None: + self.desc = desc + self.pics = pics + self.config = config + self.steps = steps + self.type = type + + def parse_python_test(path: Path) -> PythonTest: """Parse a single Python test file into PythonTest model. This will also annotate parsed python test with it's path and test type. """ - tc_steps: list[PythonTestStep] = [] - # Currently PICS and config is not configured in Python Testing - tc_pics: list = [] - tc_config: dict = {} + tc_info = __extract_tcs_info(path) - tc_desc, tc_steps = __extract_tcs_info(path) - - if not tc_desc or not tc_steps: + if not tc_info.desc or not tc_info.steps: # The file name from path tc_name = path.name.split(".")[0] raise PythonParserException( @@ -48,14 +62,20 @@ def parse_python_test(path: Path) -> PythonTest: f"or steps_{tc_name}" ) - test = PythonTest(name=tc_desc, steps=tc_steps, config=tc_config, PICS=tc_pics) + test = PythonTest( + name=tc_info.desc, steps=tc_info.steps, config=tc_info.config, PICS=tc_info.pics + ) test.path = path - test.type = PythonTestType.AUTOMATED + test.type = tc_info.type return test -def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: +def __extract_tcs_info(path: Path) -> PythonTestInfo: + # Currently PICS and config is not configured in Python Testing + tc_pics: list = [] + tc_config: dict = {} + with open(path, "r") as python_file: parsed_python_file = ast.parse(python_file.read()) classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] @@ -72,7 +92,13 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: elif "steps_" in method.name: tc_steps = __retrieve_steps(method) - return tc_desc, tc_steps + return PythonTestInfo( + desc=tc_desc, + pics=tc_pics, + config=tc_config, + steps=tc_steps, + type=PythonTestType.AUTOMATED, + ) def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py new file mode 100644 index 00000000..2ea3cbbb --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from queue import Empty, Queue +from typing import Any, Union + +from matter_yamltests.hooks import TestRunnerHooks + + +class SDKPythonTestRunnerHooks(TestRunnerHooks): + is_finished = False + results: Queue + + def updates_test(self) -> Union[dict, None]: + try: + result = self.results.get(block=False) + return result + except Empty: + return None + + def finished(self) -> bool: + return SDKPythonTestRunnerHooks.is_finished + + def __init__(self) -> None: + SDKPythonTestRunnerHooks.is_finished = False + SDKPythonTestRunnerHooks.results = Queue() + + def start(self, count: int) -> None: + self.results.put({"start": {"count": count}}) + + def stop(self, duration: int) -> None: + self.results.put({"stop": {"duration": duration}}) + self.is_finished = True + + def test_start(self, filename: str, name: str, count: int) -> None: + self.results.put( + {"test_start": {"filename": filename, "name": name, "count": count}} + ) + + def test_stop(self, exception: Exception, duration: int) -> None: + self.results.put({"test_stop": {"exception": exception, "duration": duration}}) + + def step_skipped(self, name: str, expression: str) -> None: + self.results.put({"step_skipped": {"name": name, "expression": expression}}) + + def step_start(self, name: str) -> None: + self.results.put({"step_start": {"name": name}}) + + def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: + self.results.put( + { + "step_success": { + "logger": logger, + "logs": logs, + "duration": duration, + "request": request, + } + } + ) + + def step_failure( + self, logger: Any, logs: Any, duration: int, request: Any, received: Any + ) -> None: + self.results.put( + { + "step_failure": { + "logger": logger, + "logs": logs, + "duration": duration, + "request": request, + "received": received, + } + } + ) + + def step_unknown(self) -> None: + self.results.put({"step_unknown": {}}) + + def step_manual(self) -> None: + self.results.put({"step_manual": {}}) + + def step_start_list(self) -> None: + pass diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 7861636a..75c57ddd 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -13,42 +13,141 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os import re +from multiprocessing import Process +from multiprocessing.managers import BaseManager from typing import Any, Type, TypeVar +# TODO check if this should be changed to SDK python test specific entries +from matter_chip_tool_adapter.decoder import MatterLog +from matter_yamltests.hooks import TestRunnerHooks + from app.chip_tool.chip_tool import ChipToolTestType from app.chip_tool.test_case import ChipToolTest -from app.test_engine.logger import test_engine_logger -from app.test_engine.models import TestCase, TestStep +from app.test_engine.logger import ( + CHIP_LOG_FORMAT, + CHIPTOOL_LEVEL, + logger, + test_engine_logger, +) +from app.test_engine.models import ManualVerificationTestStep, TestCase, TestStep from .python_test_models import PythonTest +from .python_testing_hooks_proxy import SDKPythonTestRunnerHooks # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") class PythonTestCase(TestCase): - """Base class for all Python based test cases. + """Base class for all Python Test based test cases. This class provides a class factory that will dynamically declare a new sub-class based on the test-type the Python test is expressing. - The PythonTest will be stored as a class property that will be used at run-time - in all instances of such subclass. + The PythonTest will be stored as a class property that will be used at run-time in all + instances of such subclass. """ python_test: PythonTest python_test_version: str + test_finished: bool + + def reset(self) -> None: + self.start_called = False + self.stop_called = False + self.test_start_called = False + self.test_stop_called = False + self.step_success_count = 0 + self.step_failure_count = 0 + self.step_unknown_count = 0 + self.__runned = 0 + + def start(self, count: int) -> None: + pass + + def stop(self, duration: int) -> None: + pass + + def test_start(self, filename: str, name: str, count: int) -> None: + pass + # Dont know if it is necessary for python testing (came from chip_tool) + # self.next_step() + + def test_stop(self, exception: Exception, duration: int) -> None: + self.current_test_step.mark_as_completed() + + def step_skipped(self, name: str, expression: str) -> None: + self.current_test_step.mark_as_not_applicable( + f"Test step skipped: {name}. {expression} == False" + ) + self.next_step() + + def step_start(self, name: str) -> None: + pass + + def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: + self.__handle_logs(logs) + self.next_step() + + def step_failure( + self, logger: Any, logs: str, duration: int, request: Any, received: Any + ) -> None: + self.__handle_logs(logs) + self.__report_failures(logger, request, received) + self.next_step() + + def step_unknown(self) -> None: + self.__runned += 1 + + def is_finished(self) -> bool: + return self.test_finished + + def __handle_logs(self, logs: Any) -> None: + for log_entry in logs or []: + if not isinstance(log_entry, MatterLog): + continue + + test_engine_logger.log( + CHIPTOOL_LEVEL, + CHIP_LOG_FORMAT.format(log_entry.module, log_entry.message), + ) + + def __report_failures(self, logger: Any, request: TestStep, received: Any) -> None: + """ + The logger from runner contains all logs entries for the test step, this method + seeks for the error entries. + """ + if not logger: + # It is expected the runner to return a PostProcessResponseResult, + # but in case of returning a different type + self.current_test_step.append_failure( + "Test Step Failure: \n " f"Expected: '<Empty>' \n Received: '<Empty>'" + ) + return + + # Iterate through the entries seeking for the errors entries + for log_entry in logger.entries or []: + if log_entry.is_error(): + # Check if the step error came from exception or not, since the message + # in exception object has more details + # TODO: There is an issue raised in SDK runner in order to improve the + # message from log_entry: + # https://github.com/project-chip/connectedhomeip/issues/28101 + if log_entry.exception: + self.current_test_step.append_failure(log_entry.exception.message) + else: + self.current_test_step.append_failure(log_entry.message) @classmethod def pics(cls) -> set[str]: - """Test Case level PICS. Read directly from parsed python test.""" + """Test Case level PICS. Read directly from parsed Python Test.""" return cls.python_test.PICS @classmethod def default_test_parameters(cls) -> dict[str, Any]: - """Python test config dict, sometimes have a nested dict with type - and default value. + """Python Testing config dict, sometimes have a nested dict with type and default value. Only defaultValue is used in this case. """ parameters = {} @@ -70,7 +169,7 @@ async def setup(self) -> None: @classmethod def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: - """Dynamically declares a subclass based on the type of Python test.""" + """Dynamically declares a subclass based on the type of Python Test test.""" case_class: Type[PythonTestCase] = PythonChipToolTestCase return case_class.__class_factory( @@ -102,7 +201,7 @@ def __class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: @staticmethod def __test_identifier(name: str) -> str: - """Find TC-XX-1.1 in Python test title. + """Find TC-XX-1.1 in Python Test title. Note some have [TC-XX-1.1] and others TC-XX-1.1 """ title_pattern = re.compile(r"(?P<title>TC-[^\s\]]*)") @@ -116,14 +215,71 @@ def __class_name(identifier: str) -> str: """Replace all non-alphanumeric characters with _ to make valid class name.""" return re.sub("[^0-9a-zA-Z]+", "_", identifier) + def run_command(self, cmd: str) -> None: + os.system(cmd) + + async def execute(self) -> None: + try: + logger.info("Running Python Test: " + self.metadata["title"]) + + # NOTE that this aproach invalidates parallel execution since test_case_instance object is shared by the class + # TODO: Same approach could work from TestCase side: create test_case_instance inside PythonTestCase to avoid using SDKPythonTestRunnerHooks + + BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) + manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") + manager.start() + + test_runner_hooks = manager.TestRunnerHooks() # type: ignore + + command = ( + "docker run -it --network host --privileged" + " -v /var/paa-root-certs:/root/paa-root-certs" + " -v /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:rw" + " -v /home/ubuntu/chip-certification-tool/backend/sdk_content/python_testing2:/root/python_testing2" + " connectedhomeip/chip-cert-bins:19771ed7101321d68b87d05201d42d00adf5368f" + " python3 python_testing2/hello_external_runner.py " + f" {self.metadata['title']}" + ) + # Start the command in a new process + p = Process(target=self.run_command, args=(command,)) + p.start() + + while ((update := test_runner_hooks.updates_test()) is not None) or ( + not test_runner_hooks.finished() + ): + if not update: + continue + + def handle_update(update: dict) -> None: + def call_function(obj, func_name: str, kwargs) -> None: # type: ignore + func = getattr(obj, func_name, None) + if not func: + raise AttributeError( + f"{func_name} is not a method of {obj}" + ) + if not callable(func): + raise TypeError(f"{func_name} is not callable") + # Call the method with the unpacked keyword arguments. + func(**kwargs) + + for func_name, kwargs in update.items(): + call_function(self, func_name, kwargs) + + handle_update(update) + + finally: + pass + + async def cleanup(self) -> None: + logger.info("Test Cleanup") + class PythonChipToolTestCase(PythonTestCase, ChipToolTest): - """Automated Python test cases.""" + """Automated Python test cases using chip-tool.""" test_type = ChipToolTestType.PYTHON_TEST def create_test_steps(self) -> None: - self.test_steps = [TestStep("Start Python test")] for step in self.python_test.steps: python_test_step = TestStep(step.label) self.test_steps.append(python_test_step) diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py index c16d10d4..5945505d 100644 --- a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -17,6 +17,8 @@ from loguru import logger +from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH + from .models.python_test_folder import PythonTestFolder from .models.python_test_parser import PythonParserException, parse_python_test from .models.test_declarations import ( @@ -27,7 +29,7 @@ from .models.test_suite import SuiteType ### -# This file hosts logic load and parse Python test-cases, located in +# This file hosts logic to load and parse Python test cases, located in # `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. # The `sdk` sub-folder here is automatically maintained using the # `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. @@ -36,9 +38,7 @@ # - Automated ### -SDK_PYTHON_TEST_PATH = Path( - "/app/backend/test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk" -) +SDK_PYTHON_TEST_PATH = SDK_CHECKOUT_PATH / Path("python_testing/scripts/sdk") SDK_PYTHON_TEST_FOLDER = PythonTestFolder( path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*" ) From 879b7de97cfdd25edaa8e813c840c3fd3018ee26 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Fri, 10 Nov 2023 18:27:19 +0000 Subject: [PATCH 16/45] Fix code violation and add code to chip_tool for python runner --- app/chip_tool/chip_tool.py | 43 +++++++-------- app/chip_tool/test_case.py | 1 + .../models/python_test_parser.py | 2 +- .../python_testing/models/test_case.py | 54 +++++++++---------- 4 files changed, 46 insertions(+), 54 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index ff88efb7..ccafa8ee 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -266,9 +266,6 @@ async def start_chip_server( elif test_type == ChipToolTestType.CHIP_APP: prefix = CHIP_APP_EXE command = ["--interactive", "--port 9002"] - elif test_type == ChipToolTestType.PYTHON_TEST: - # TODO - To be implemented - pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") @@ -349,46 +346,47 @@ def __get_gateway_ip(self) -> str: .get(DOCKER_GATEWAY_KEY, "") ) - async def start_container( - self, test_type: ChipToolTestType, use_paa_certs: bool = False - ) -> None: - """Creates the chip-tool container. - - Returns only when the container is created and all chip-tool services start. + async def start_container_no_server(self) -> None: + """ + Creates the chip-tool container without any service running + (ChipTool or ChipApp). """ - - if self.is_running(): - self.logger.info( - "chip-tool container already running, no need to start a new container" - ) - return - # Ensure there's no existing container running using the same name. self.__destroy_existing_container() - # Async return when the container is running self.__chip_tool_container = await container_manager.create_container( self.image_tag, self.run_parameters ) - # Reset any previous states self.__last_exec_id = None self.__pics_file_created = False - # Generate new random node id for the DUT self.__reset_node_id() self.logger.info(f"New Node Id generated: {hex(self.node_id)}") - self.logger.info( f""" chip-tool started: {self.container_name} with configuration: {self.run_parameters} """ ) - # Server started is false after spinning up a new container. self.__server_started = False + async def start_container( + self, test_type: ChipToolTestType, use_paa_certs: bool = False + ) -> None: + """Creates the chip-tool container. + + Returns only when the container is created and all chip-tool services start. + """ + if self.is_running(): + self.logger.info( + "chip-tool container already running, no need to start a new container" + ) + return + + await self.start_container_no_server() + web_socket_config = WebSocketRunnerConfig() web_socket_config.server_address = self.__get_gateway_ip() self.__test_harness_runner = WebSocketRunner(config=web_socket_config) @@ -602,9 +600,6 @@ async def run_test( adapter = ChipToolAdapter.Adapter(parser_config.definitions) elif test_type == ChipToolTestType.CHIP_APP: adapter = ChipAppAdapter.Adapter(parser_config.definitions) - elif test_type == ChipToolTestType.PYTHON_TEST: - # TODO - run_test() for python_test to be implemented - pass else: raise ChipToolUnknownTestType(f"Unsupported Test Type: {test_type}") diff --git a/app/chip_tool/test_case.py b/app/chip_tool/test_case.py index db93970e..a3bbf365 100644 --- a/app/chip_tool/test_case.py +++ b/app/chip_tool/test_case.py @@ -225,6 +225,7 @@ async def setup(self) -> None: ) self.chip_tool = ChipTool() + await self.chip_tool.start_container_no_server() # Use test engine logger to log all events to test run. self.chip_tool.logger = test_engine_logger diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index e5f6d9b9..7dec9c13 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -15,7 +15,7 @@ # import ast from pathlib import Path -from typing import List, Tuple +from typing import List from .python_test_models import PythonTest, PythonTestStep, PythonTestType diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 75c57ddd..c15d569c 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -15,23 +15,21 @@ # import os import re -from multiprocessing import Process from multiprocessing.managers import BaseManager from typing import Any, Type, TypeVar -# TODO check if this should be changed to SDK python test specific entries from matter_chip_tool_adapter.decoder import MatterLog -from matter_yamltests.hooks import TestRunnerHooks -from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.chip_tool import ChipTool, ChipToolTestType from app.chip_tool.test_case import ChipToolTest +from app.models import TestCaseExecution from app.test_engine.logger import ( CHIP_LOG_FORMAT, CHIPTOOL_LEVEL, logger, test_engine_logger, ) -from app.test_engine.models import ManualVerificationTestStep, TestCase, TestStep +from app.test_engine.models import TestCase, TestStep from .python_test_models import PythonTest from .python_testing_hooks_proxy import SDKPythonTestRunnerHooks @@ -39,6 +37,11 @@ # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") +# Command line params +RUNNER_CLASS = "external_runner.py" +RUNNER_CLASS_PATH = "./testing2/" +EXECUTABLE = "python3" + class PythonTestCase(TestCase): """Base class for all Python Test based test cases. @@ -46,14 +49,18 @@ class PythonTestCase(TestCase): This class provides a class factory that will dynamically declare a new sub-class based on the test-type the Python test is expressing. - The PythonTest will be stored as a class property that will be used at run-time in all - instances of such subclass. + The PythonTest will be stored as a class property that will be used at run-time + in all instances of such subclass. """ python_test: PythonTest python_test_version: str test_finished: bool + def __init__(self, test_case_execution: TestCaseExecution) -> None: + super().__init__(test_case_execution=test_case_execution) + self.chip_tool: ChipTool + def reset(self) -> None: self.start_called = False self.stop_called = False @@ -123,7 +130,7 @@ def __report_failures(self, logger: Any, request: TestStep, received: Any) -> No # It is expected the runner to return a PostProcessResponseResult, # but in case of returning a different type self.current_test_step.append_failure( - "Test Step Failure: \n " f"Expected: '<Empty>' \n Received: '<Empty>'" + "Test Step Failure: \n Expected: '<Empty>' \n Received: '<Empty>'" ) return @@ -147,7 +154,8 @@ def pics(cls) -> set[str]: @classmethod def default_test_parameters(cls) -> dict[str, Any]: - """Python Testing config dict, sometimes have a nested dict with type and default value. + """Python Testing config dict, sometimes have a nested dict with type and + default value. Only defaultValue is used in this case. """ parameters = {} @@ -164,6 +172,9 @@ async def setup(self) -> None: test_engine_logger.info(f"Python Test Version: {self.python_test_version}") try: await super().setup() + self.chip_tool = ChipTool() + await self.chip_tool.start_container_no_server() + assert self.chip_tool.is_running() except NotImplementedError: pass @@ -221,29 +232,14 @@ def run_command(self, cmd: str) -> None: async def execute(self) -> None: try: logger.info("Running Python Test: " + self.metadata["title"]) - - # NOTE that this aproach invalidates parallel execution since test_case_instance object is shared by the class - # TODO: Same approach could work from TestCase side: create test_case_instance inside PythonTestCase to avoid using SDKPythonTestRunnerHooks - BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") manager.start() - test_runner_hooks = manager.TestRunnerHooks() # type: ignore - - command = ( - "docker run -it --network host --privileged" - " -v /var/paa-root-certs:/root/paa-root-certs" - " -v /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:rw" - " -v /home/ubuntu/chip-certification-tool/backend/sdk_content/python_testing2:/root/python_testing2" - " connectedhomeip/chip-cert-bins:19771ed7101321d68b87d05201d42d00adf5368f" - " python3 python_testing2/hello_external_runner.py " - f" {self.metadata['title']}" + runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS + result = self.chip_tool.send_command( + f"{runner_class} {self.metadata['title']}", prefix=EXECUTABLE ) - # Start the command in a new process - p = Process(target=self.run_command, args=(command,)) - p.start() - while ((update := test_runner_hooks.updates_test()) is not None) or ( not test_runner_hooks.finished() ): @@ -251,7 +247,7 @@ async def execute(self) -> None: continue def handle_update(update: dict) -> None: - def call_function(obj, func_name: str, kwargs) -> None: # type: ignore + def call_function(obj, func_name, kwargs) -> None: # type: ignore func = getattr(obj, func_name, None) if not func: raise AttributeError( @@ -266,12 +262,12 @@ def call_function(obj, func_name: str, kwargs) -> None: # type: ignore call_function(self, func_name, kwargs) handle_update(update) - finally: pass async def cleanup(self) -> None: logger.info("Test Cleanup") + self.chip_tool.destroy_device() class PythonChipToolTestCase(PythonTestCase, ChipToolTest): From 5872265473749e69ed17ae2453942314fb8113f7 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Fri, 10 Nov 2023 18:36:11 +0000 Subject: [PATCH 17/45] Fix code violation --- .../sdk_tests/support/python_testing/models/test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index c15d569c..f36061ef 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -237,7 +237,7 @@ async def execute(self) -> None: manager.start() test_runner_hooks = manager.TestRunnerHooks() # type: ignore runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS - result = self.chip_tool.send_command( + self.chip_tool.send_command( f"{runner_class} {self.metadata['title']}", prefix=EXECUTABLE ) while ((update := test_runner_hooks.updates_test()) is not None) or ( From 8a89c18bcf5ce19bf80ac48b88b0c5f8bc533755 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Mon, 13 Nov 2023 14:09:14 +0000 Subject: [PATCH 18/45] Changes after code review --- app/chip_tool/chip_tool.py | 2 +- app/chip_tool/test_case.py | 1 - test_collections/sdk_tests/__init__.py | 5 +---- .../python_testing/models/python_testing_hooks_proxy.py | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index ccafa8ee..2b184091 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -348,7 +348,7 @@ def __get_gateway_ip(self) -> str: async def start_container_no_server(self) -> None: """ - Creates the chip-tool container without any service running + Creates the chip-tool container without any server running (ChipTool or ChipApp). """ # Ensure there's no existing container running using the same name. diff --git a/app/chip_tool/test_case.py b/app/chip_tool/test_case.py index a3bbf365..db93970e 100644 --- a/app/chip_tool/test_case.py +++ b/app/chip_tool/test_case.py @@ -225,7 +225,6 @@ async def setup(self) -> None: ) self.chip_tool = ChipTool() - await self.chip_tool.start_container_no_server() # Use test engine logger to log all events to test run. self.chip_tool.logger = test_engine_logger diff --git a/test_collections/sdk_tests/__init__.py b/test_collections/sdk_tests/__init__.py index 1460bd59..16c1b31e 100644 --- a/test_collections/sdk_tests/__init__.py +++ b/test_collections/sdk_tests/__init__.py @@ -13,8 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from .support.python_testing import sdk_python_collection from .support.yaml_tests import custom_collection, sdk_collection - -# TODO The Python test support implementation is ongoing. -# Uncomment line below when the implementation is done. -# from .support.python_testing import sdk_python_collection diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index 2ea3cbbb..c26078da 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -23,14 +23,14 @@ class SDKPythonTestRunnerHooks(TestRunnerHooks): is_finished = False results: Queue - def updates_test(self) -> Union[dict, None]: + def update_test(self) -> Union[dict, None]: try: result = self.results.get(block=False) return result except Empty: return None - def finished(self) -> bool: + def is_finished(self) -> bool: return SDKPythonTestRunnerHooks.is_finished def __init__(self) -> None: From 118cc896c27065a8378ef4fe09182b63ff094514 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Mon, 13 Nov 2023 14:24:12 +0000 Subject: [PATCH 19/45] Removed unsued code --- .../support/python_testing/models/test_case.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index f36061ef..17859d98 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -61,16 +61,6 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: super().__init__(test_case_execution=test_case_execution) self.chip_tool: ChipTool - def reset(self) -> None: - self.start_called = False - self.stop_called = False - self.test_start_called = False - self.test_stop_called = False - self.step_success_count = 0 - self.step_failure_count = 0 - self.step_unknown_count = 0 - self.__runned = 0 - def start(self, count: int) -> None: pass @@ -240,8 +230,8 @@ async def execute(self) -> None: self.chip_tool.send_command( f"{runner_class} {self.metadata['title']}", prefix=EXECUTABLE ) - while ((update := test_runner_hooks.updates_test()) is not None) or ( - not test_runner_hooks.finished() + while ((update := test_runner_hooks.update_test()) is not None) or ( + not test_runner_hooks.is_finished() ): if not update: continue From b914e195d0abe92bec283d6280447dde3c5a8581 Mon Sep 17 00:00:00 2001 From: Gladyston Franca <gladyston@apple.com> Date: Mon, 13 Nov 2023 13:03:36 -0300 Subject: [PATCH 20/45] Few tweeks based on observation running end to end. --- .../python_testing/models/test_case.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 17859d98..0054838e 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -15,6 +15,7 @@ # import os import re +from asyncio import sleep from multiprocessing.managers import BaseManager from typing import Any, Type, TypeVar @@ -55,7 +56,6 @@ class PythonTestCase(TestCase): python_test: PythonTest python_test_version: str - test_finished: bool def __init__(self, test_case_execution: TestCaseExecution) -> None: super().__init__(test_case_execution=test_case_execution) @@ -65,7 +65,8 @@ def start(self, count: int) -> None: pass def stop(self, duration: int) -> None: - pass + if not self.test_stop_called: + self.current_test_step.mark_as_completed() def test_start(self, filename: str, name: str, count: int) -> None: pass @@ -73,6 +74,7 @@ def test_start(self, filename: str, name: str, count: int) -> None: # self.next_step() def test_stop(self, exception: Exception, duration: int) -> None: + self.test_stop_called = True self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: @@ -98,9 +100,6 @@ def step_failure( def step_unknown(self) -> None: self.__runned += 1 - def is_finished(self) -> bool: - return self.test_finished - def __handle_logs(self, logs: Any) -> None: for log_entry in logs or []: if not isinstance(log_entry, MatterLog): @@ -227,9 +226,14 @@ async def execute(self) -> None: manager.start() test_runner_hooks = manager.TestRunnerHooks() # type: ignore runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS + # TODO Ignoring stream from docker execution until client impl is done self.chip_tool.send_command( - f"{runner_class} {self.metadata['title']}", prefix=EXECUTABLE - ) + f"{runner_class} {self.metadata['title']}", + prefix=EXECUTABLE, + is_stream=True, + is_socket=False, + ).output + while ((update := test_runner_hooks.update_test()) is not None) or ( not test_runner_hooks.is_finished() ): @@ -252,6 +256,7 @@ def call_function(obj, func_name, kwargs) -> None: # type: ignore call_function(self, func_name, kwargs) handle_update(update) + await sleep(0.1) finally: pass From ba9c21ee830834395331d210aaffb572b8ece242 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Tue, 14 Nov 2023 16:23:28 +0000 Subject: [PATCH 21/45] Created top level classes for test models --- app/chip_tool/test_case.py | 2 +- .../python_tests/test_python_test_case.py | 28 ++++---- .../test_sdk_python_collection.py | 6 +- .../yaml_tests/test_sdk_yaml_collection.py | 17 ++--- app/tests/yaml_tests/test_test_case.py | 72 ++++++++++--------- app/tests/yaml_tests/test_yaml_parser.py | 34 ++++----- .../support/models/th_test_models.py | 49 +++++++++++++ .../models/python_test_models.py | 30 ++------ .../models/python_test_parser.py | 21 +++--- .../models/test_declarations.py | 4 +- .../support/yaml_tests/models/test_case.py | 16 +++-- .../yaml_tests/models/test_declarations.py | 5 +- .../yaml_tests/models/yaml_test_models.py | 34 ++------- .../yaml_tests/models/yaml_test_parser.py | 14 ++-- .../support/yaml_tests/sdk_yaml_tests.py | 6 +- 15 files changed, 183 insertions(+), 155 deletions(-) create mode 100644 test_collections/sdk_tests/support/models/th_test_models.py diff --git a/app/chip_tool/test_case.py b/app/chip_tool/test_case.py index db93970e..d1881c11 100644 --- a/app/chip_tool/test_case.py +++ b/app/chip_tool/test_case.py @@ -108,7 +108,7 @@ def test_start(self, filename: str, name: str, count: int) -> None: # since there is step execute outside runner context self.next_step() - def test_stop(self, exception: Exception, duration: int) -> None: + def test_stop(self, duration: int) -> None: self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index 0a3cbb2f..98cd8b8a 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -25,11 +25,13 @@ from app.chip_tool.test_case import TestError from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger +from test_collections.sdk_tests.support.models.th_test_models import ( + THTestStep, + THTestType, +) from test_collections.sdk_tests.support.python_testing.models import PythonTestCase from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( PythonTest, - PythonTestStep, - PythonTestType, ) from test_collections.sdk_tests.support.python_testing.models.test_case import ( PythonChipToolTestCase, @@ -43,8 +45,8 @@ def python_test_instance( "param1": "value1", "param2": {"type": "config_type", "defaultValue": "value2"}, }, - steps: list[PythonTestStep] = [], - type: PythonTestType = PythonTestType.AUTOMATED, + steps: list[THTestStep] = [], + type: THTestType = THTestType.AUTOMATED, path: Optional[Path] = None, ) -> PythonTest: return PythonTest( @@ -127,7 +129,7 @@ def test_python_test_case_class_default_test_parameters() -> None: def test_automated_test_case_class_factory_subclass_mapping() -> None: """Test Automated tests are created as a subclass of PythonChipToolTestCase.""" - test = python_test_instance(type=PythonTestType.AUTOMATED) + test = python_test_instance(type=THTestType.AUTOMATED) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" ) @@ -176,7 +178,7 @@ def test_class_factory_test_class_name() -> None: def test_test_type_for_automated_tests() -> None: """Test that automated tests are set to use chip-tool""" - test = python_test_instance(type=PythonTestType.AUTOMATED) + test = python_test_instance(type=THTestType.AUTOMATED) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" ) @@ -191,7 +193,7 @@ async def test_python_version_logging() -> None: Note that since `chip-tool` is not setup, we except the TestError raised. """ - for type in list(PythonTestType): + for type in list(THTestType): test = python_test_instance(type=type) test_python_version = "PythonVersionTest" case_class: Type[PythonTestCase] = PythonTestCase.class_factory( @@ -211,9 +213,9 @@ async def test_python_version_logging() -> None: def test_normal_steps_for_python_tests() -> None: - """Test that non-manual tests include enabled steps.""" - for type in list(PythonTestType): - test_step = PythonTestStep(label="Step1") + """Test that python tests include enabled steps.""" + for type in list(THTestType): + test_step = THTestStep(label="Step1") test = python_test_instance(type=type, steps=[test_step]) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" @@ -225,9 +227,9 @@ def test_normal_steps_for_python_tests() -> None: def test_multiple_steps_for_python_tests() -> None: - """Test that non-manual tests multiple enabled steps are all included.""" - for type in list(PythonTestType): - test_step = PythonTestStep(label="StepN") + """Test that python tests multiple enabled steps are all included.""" + for type in list(THTestType): + test_step = THTestStep(label="StepN") no_steps = 5 test = python_test_instance(type=type, steps=([test_step] * no_steps)) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index ff6534fa..2bb5b5d1 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -24,7 +24,7 @@ PythonTestFolder, ) from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( - PythonTestType, + THTestType, ) from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( PythonCaseDeclaration, @@ -72,9 +72,9 @@ def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> automated_suite = python_test_collection.test_suites["Python Testing Suite"] assert len(automated_suite.test_cases) == expected_automated_test_cases - type_count = dict.fromkeys(PythonTestType, 0) + type_count = dict.fromkeys(THTestType, 0) for test_case in automated_suite.test_cases.values(): assert isinstance(test_case, PythonCaseDeclaration) type_count[test_case.test_type] += 1 - assert type_count[PythonTestType.AUTOMATED] == expected_automated_test_cases + assert type_count[THTestType.AUTOMATED] == expected_automated_test_cases diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index b4c3ca47..f777d0fc 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -27,12 +27,13 @@ YamlTestFolder, ) from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( - YamlTestType, + THTestType, ) from test_collections.sdk_tests.support.yaml_tests.sdk_yaml_tests import ( sdk_yaml_test_collection, ) + @pytest.fixture def yaml_collection() -> YamlCollectionDeclaration: test_sdk_yaml_path = Path(__file__).parent / "test_yamls" @@ -66,7 +67,7 @@ def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: assert len(manual_suite.test_cases) == expected_manual_test_cases for test_case in manual_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) - assert test_case.test_type == YamlTestType.MANUAL + assert test_case.test_type == THTestType.MANUAL def test_automated_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -83,15 +84,15 @@ def test_automated_suite(yaml_collection: YamlCollectionDeclaration) -> None: == expected_automated_test_cases + expected_semi_automated_test_cases ) - type_count = dict.fromkeys(YamlTestType, 0) + type_count = dict.fromkeys(THTestType, 0) for test_case in automated_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) type_count[test_case.test_type] += 1 - assert type_count[YamlTestType.AUTOMATED] == expected_automated_test_cases - assert type_count[YamlTestType.SEMI_AUTOMATED] == expected_semi_automated_test_cases - assert type_count[YamlTestType.SIMULATED] == expected_simulated_test_cases - assert type_count[YamlTestType.MANUAL] == expected_manual_test_cases + assert type_count[THTestType.AUTOMATED] == expected_automated_test_cases + assert type_count[THTestType.SEMI_AUTOMATED] == expected_semi_automated_test_cases + assert type_count[THTestType.SIMULATED] == expected_simulated_test_cases + assert type_count[THTestType.MANUAL] == expected_manual_test_cases def test_simulated_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -102,4 +103,4 @@ def test_simulated_suite(yaml_collection: YamlCollectionDeclaration) -> None: assert len(simulated_suite.test_cases) == expected_simulated_test_cases for test_case in simulated_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) - assert test_case.test_type == YamlTestType.SIMULATED + assert test_case.test_type == THTestType.SIMULATED diff --git a/app/tests/yaml_tests/test_test_case.py b/app/tests/yaml_tests/test_test_case.py index a4b03998..90a51900 100644 --- a/app/tests/yaml_tests/test_test_case.py +++ b/app/tests/yaml_tests/test_test_case.py @@ -24,6 +24,10 @@ from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger from app.test_engine.models.manual_test_case import ManualVerificationTestStep +from test_collections.sdk_tests.support.models.th_test_models import ( + THTestStep, + THTestType, +) from test_collections.sdk_tests.support.yaml_tests.models import YamlTestCase from test_collections.sdk_tests.support.yaml_tests.models.test_case import ( YamlChipToolTestCase, @@ -33,8 +37,6 @@ ) from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( YamlTest, - YamlTestStep, - YamlTestType, ) @@ -45,8 +47,8 @@ def yaml_test_instance( "param1": "value1", "param2": {"type": "config_type", "defaultValue": "value2"}, }, - tests: list[YamlTestStep] = [], - type: YamlTestType = YamlTestType.AUTOMATED, + tests: list[THTestStep] = [], + type: THTestType = THTestType.AUTOMATED, path: Optional[Path] = None, ) -> YamlTest: return YamlTest( @@ -128,7 +130,7 @@ def test_yaml_test_case_class_default_test_parameters() -> None: def test_manual_test_case_class_factory_subclass_mapping() -> None: """Test Manual tests are created as a subclass of YamlManualTestCase.""" - test = yaml_test_instance(type=YamlTestType.MANUAL) + test = yaml_test_instance(type=THTestType.MANUAL) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -138,7 +140,7 @@ def test_manual_test_case_class_factory_subclass_mapping() -> None: def test_automated_test_case_class_factory_subclass_mapping() -> None: """Test Automated tests are created as a subclass of YamlChipToolTestCase.""" - test = yaml_test_instance(type=YamlTestType.AUTOMATED) + test = yaml_test_instance(type=THTestType.AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -148,7 +150,7 @@ def test_automated_test_case_class_factory_subclass_mapping() -> None: def test_semi_automated_test_case_class_factory_subclass_mapping() -> None: """Test Semi-Automated tests are created as a subclass of YamlSemiAutomatedChipToolTestCase.""" - test = yaml_test_instance(type=YamlTestType.SEMI_AUTOMATED) + test = yaml_test_instance(type=THTestType.SEMI_AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -158,7 +160,7 @@ def test_semi_automated_test_case_class_factory_subclass_mapping() -> None: def test_simulated_test_case_class_factory_subclass_mapping() -> None: """Test Simulated tests are created as a subclass of YamlSimulatedTestCase.""" - test = yaml_test_instance(type=YamlTestType.SIMULATED) + test = yaml_test_instance(type=THTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -168,7 +170,7 @@ def test_simulated_test_case_class_factory_subclass_mapping() -> None: def test_incomplete_test_case_class_factory_subclass_mapping() -> None: """Test Semi-Automated tests are created as a subclass of YamlSimulatedTestCase.""" - test = yaml_test_instance(type=YamlTestType.SIMULATED) + test = yaml_test_instance(type=THTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -219,13 +221,13 @@ def test_class_factory_test_title_semi_automated() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1', use it as metadata title, and append (Semi-automated) when appropriate.""" - for type in list(YamlTestType): + for type in list(THTestType): test = yaml_test_instance(type=type, name="TC-AB-1.2") case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) - if type == YamlTestType.SEMI_AUTOMATED: + if type == THTestType.SEMI_AUTOMATED: assert test.name in case_class.metadata["title"] assert "(Semi-automated)" in case_class.metadata["title"] else: @@ -235,12 +237,12 @@ def test_class_factory_test_title_semi_automated() -> None: def test_class_factory_test_title_steps_disabled() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1', use it as metadata title, and (Steps Disabled) when some but not all steps are disabled.""" - disabled_step = YamlTestStep(label="Step1", disabled=True) - enabled_step = YamlTestStep(label="Step2", disabled=False) + disabled_step = THTestStep(label="Step1", disabled=True) + enabled_step = THTestStep(label="Step2", disabled=False) for type in [ - YamlTestType.AUTOMATED, - YamlTestType.SEMI_AUTOMATED, - YamlTestType.SIMULATED, + THTestType.AUTOMATED, + THTestType.SEMI_AUTOMATED, + THTestType.SIMULATED, ]: test = yaml_test_instance( type=type, name="TC-AB-1.2", tests=[disabled_step, enabled_step] @@ -261,10 +263,10 @@ def test_steps_in_manual_yaml_test_case() -> None: - verification details are optional but passed when present in yaml """ steps = [ - YamlTestStep(label="Step1"), - YamlTestStep(label="Step2Verification", verification="Verification String"), + THTestStep(label="Step1"), + THTestStep(label="Step2Verification", verification="Verification String"), ] - test = yaml_test_instance(type=YamlTestType.MANUAL, tests=steps) + test = yaml_test_instance(type=THTestType.MANUAL, tests=steps) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -281,7 +283,7 @@ def test_steps_in_manual_yaml_test_case() -> None: def test_test_type_for_automated_tests() -> None: """Test that automated tests are set to use chip-tool""" - test = yaml_test_instance(type=YamlTestType.AUTOMATED) + test = yaml_test_instance(type=THTestType.AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -292,7 +294,7 @@ def test_test_type_for_automated_tests() -> None: def test_test_type_for_simulated_tests() -> None: """Test that simulated tests are set to use chip-app""" - test = yaml_test_instance(type=YamlTestType.SIMULATED) + test = yaml_test_instance(type=THTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -307,7 +309,7 @@ async def test_yaml_version_logging() -> None: Note that since `chip-tool` is not setup, we except the TestError raised. """ - for type in list(YamlTestType): + for type in list(THTestType): test = yaml_test_instance(type=type) test_yaml_version = "YamlVersionTest" case_class: Type[YamlTestCase] = YamlTestCase.class_factory( @@ -327,7 +329,7 @@ async def test_yaml_version_logging() -> None: def test_default_first_steps_for_yaml_chip_tool_test_case() -> None: - test = yaml_test_instance(type=YamlTestType.AUTOMATED, tests=[]) + test = yaml_test_instance(type=THTestType.AUTOMATED, tests=[]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -337,7 +339,7 @@ def test_default_first_steps_for_yaml_chip_tool_test_case() -> None: def test_no_default_first_steps_for_yaml_simulated_test_case() -> None: - test = yaml_test_instance(type=YamlTestType.SIMULATED, tests=[]) + test = yaml_test_instance(type=THTestType.SIMULATED, tests=[]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -348,10 +350,10 @@ def test_no_default_first_steps_for_yaml_simulated_test_case() -> None: def test_disabled_steps_for_non_manual_test() -> None: """Test that non-manual tests skip disabled steps.""" - for type in list(YamlTestType): - if type == YamlTestType.MANUAL: + for type in list(THTestType): + if type == THTestType.MANUAL: continue - test_step = YamlTestStep(label="Disabled Step", disabled=True) + test_step = THTestStep(label="Disabled Step", disabled=True) test = yaml_test_instance(type=type, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" @@ -363,10 +365,10 @@ def test_disabled_steps_for_non_manual_test() -> None: def test_normal_steps_for_non_manual_tests() -> None: """Test that non-manual tests include enabled steps.""" - for type in list(YamlTestType): - if type == YamlTestType.MANUAL: + for type in list(THTestType): + if type == THTestType.MANUAL: continue - test_step = YamlTestStep(label="Step1") + test_step = THTestStep(label="Step1") test = yaml_test_instance(type=type, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" @@ -379,10 +381,10 @@ def test_normal_steps_for_non_manual_tests() -> None: def test_multiple_steps_for_non_manual() -> None: """Test that non-manual tests multiple enabled steps are all included.""" - for type in list(YamlTestType): - if type == YamlTestType.MANUAL: + for type in list(THTestType): + if type == THTestType.MANUAL: continue - test_step = YamlTestStep(label="StepN") + test_step = THTestStep(label="StepN") no_steps = 5 test = yaml_test_instance(type=type, tests=([test_step] * no_steps)) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( @@ -397,13 +399,13 @@ def test_multiple_steps_for_non_manual() -> None: def test_prompt_steps_for_yaml_chip_tool_test_case() -> None: - test_step = YamlTestStep( + test_step = THTestStep( label="Step1", command="UserPrompt", verification="Verify that This happened", ) - test = yaml_test_instance(type=YamlTestType.AUTOMATED, tests=[test_step]) + test = yaml_test_instance(type=THTestType.AUTOMATED, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) diff --git a/app/tests/yaml_tests/test_yaml_parser.py b/app/tests/yaml_tests/test_yaml_parser.py index 69be5216..2c39938e 100644 --- a/app/tests/yaml_tests/test_yaml_parser.py +++ b/app/tests/yaml_tests/test_yaml_parser.py @@ -20,9 +20,9 @@ from pydantic.error_wrappers import ValidationError from app.tests.yaml_tests.test_test_case import yaml_test_instance -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( - YamlTestStep, - YamlTestType, +from test_collections.sdk_tests.support.models.th_test_models import ( + THTestStep, + THTestType, ) from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_parser import ( YamlParserException, @@ -97,54 +97,54 @@ def test_yaml_file_parser() -> None: def test_test_type_all_disabled_steps() -> None: - disabled_step = YamlTestStep(label="Disabled Test Step", disabled=True) + disabled_step = THTestStep(label="Disabled Test Step", disabled=True) five_disabled_steps_test = yaml_test_instance(tests=[disabled_step] * 5) type = _test_type(five_disabled_steps_test) - assert type == YamlTestType.MANUAL + assert type == THTestType.MANUAL # simulated in path overrides test type to simulated five_disabled_steps_test.path = Path("TC_XX_Simulated.yaml") type = _test_type(five_disabled_steps_test) - assert type == YamlTestType.SIMULATED + assert type == THTestType.SIMULATED def test_test_type_some_disabled_steps() -> None: - disabled_step = YamlTestStep(label="Disabled Test Step", disabled=True) - enabled_step = YamlTestStep(label="Enabled Test Step", disabled=False) + disabled_step = THTestStep(label="Disabled Test Step", disabled=True) + enabled_step = THTestStep(label="Enabled Test Step", disabled=False) test = yaml_test_instance(tests=[disabled_step, enabled_step]) type = _test_type(test) - assert type == YamlTestType.AUTOMATED + assert type == THTestType.AUTOMATED # simulated in path overrides test type to simulated test.path = Path("TC_XX_Simulated.yaml") type = _test_type(test) - assert type == YamlTestType.SIMULATED + assert type == THTestType.SIMULATED def test_test_type_all_enabled_steps_no_prompts() -> None: - enabled_step = YamlTestStep(label="Enabled Test Step") + enabled_step = THTestStep(label="Enabled Test Step") five_enabled_steps_test = yaml_test_instance(tests=[enabled_step] * 5) type = _test_type(five_enabled_steps_test) - assert type == YamlTestType.AUTOMATED + assert type == THTestType.AUTOMATED # simulated in path overrides test type to simulated five_enabled_steps_test.path = Path("TC_XX_Simulated.yaml") type = _test_type(five_enabled_steps_test) - assert type == YamlTestType.SIMULATED + assert type == THTestType.SIMULATED def test_test_type_all_enabled_steps_some_prompts() -> None: - enabled_step = YamlTestStep(label="Enable Test Step") - prompt_step = YamlTestStep(label="Prompt Test Step", command="UserPrompt") + enabled_step = THTestStep(label="Enable Test Step") + prompt_step = THTestStep(label="Prompt Test Step", command="UserPrompt") test = yaml_test_instance(tests=[enabled_step, prompt_step]) type = _test_type(test) - assert type == YamlTestType.SEMI_AUTOMATED + assert type == THTestType.SEMI_AUTOMATED # simulated in path overrides test type to simulated test.path = Path("TC_XX_Simulated.yaml") type = _test_type(test) - assert type == YamlTestType.SIMULATED + assert type == THTestType.SIMULATED diff --git a/test_collections/sdk_tests/support/models/th_test_models.py b/test_collections/sdk_tests/support/models/th_test_models.py new file mode 100644 index 00000000..e4f6675f --- /dev/null +++ b/test_collections/sdk_tests/support/models/th_test_models.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import Enum +from pathlib import Path +from typing import Any, Optional + +from pydantic import BaseModel + +### +# This file declares YAML models that are used to parse the YAML Test Cases. +### + + +class THTestType(Enum): + AUTOMATED = 0 + SEMI_AUTOMATED = 1 + MANUAL = 2 + SIMULATED = 3 + + +class THTestStep(BaseModel): + label: str + PICS: Optional[str] = None + verification: Optional[str] = None + command: Optional[str] + disabled: bool = False + arguments: Optional[dict[str, Any]] + + +class THTest(BaseModel): + name: str + PICS: set[str] = set() + config: dict[str, Any] + steps: list[THTestStep] + type: THTestType = THTestType.MANUAL + path: Optional[Path] diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index 51a6860e..6cddb0ff 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -13,34 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from enum import Enum -from pathlib import Path -from typing import Any, Optional +from typing import Any -from pydantic import BaseModel +from ...models.th_test_models import THTest, THTestType ### # This file declares Python test models that are used to parse the Python Test Cases. ### -class PythonTestType(Enum): - AUTOMATED = 0 - - -class PythonTestStep(BaseModel): - label: str - PICS: Optional[str] = None - verification: Optional[str] = None - command: Optional[str] - disabled: bool = False - arguments: Optional[dict[str, Any]] - - -class PythonTest(BaseModel): - name: str - PICS: set[str] = set() - config: dict[str, Any] - steps: list[PythonTestStep] - type: PythonTestType = PythonTestType.AUTOMATED - path: Optional[Path] +class PythonTest(THTest): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.type: THTestType.MANUAL # type: ignore diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 7dec9c13..1503535d 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -17,7 +17,12 @@ from pathlib import Path from typing import List -from .python_test_models import PythonTest, PythonTestStep, PythonTestType +from test_collections.sdk_tests.support.models.th_test_models import ( + THTestStep, + THTestType, +) + +from .python_test_models import PythonTest ARG_STEP_DESCRIPTION_INDEX = 1 KEYWORD_IS_COMISSIONING_INDEX = 0 @@ -37,8 +42,8 @@ def __init__( desc: str, pics: list, config: dict, - steps: list[PythonTestStep], - type: PythonTestType, + steps: list[THTestStep], + type: THTestType, ) -> None: self.desc = desc self.pics = pics @@ -82,7 +87,7 @@ def __extract_tcs_info(path: Path) -> PythonTestInfo: # Get TC description and TC steps from python test file tc_desc: str = "" - tc_steps: List[PythonTestStep] = [] + tc_steps: List[THTestStep] = [] for class_ in classes: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] @@ -97,12 +102,12 @@ def __extract_tcs_info(path: Path) -> PythonTestInfo: pics=tc_pics, config=tc_config, steps=tc_steps, - type=PythonTestType.AUTOMATED, + type=THTestType.AUTOMATED, ) -def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: - python_steps: List[PythonTestStep] = [] +def __retrieve_steps(method: ast.FunctionDef) -> List[THTestStep]: + python_steps: List[THTestStep] = [] for step in method.body[BODY_INDEX].value.elts: # type: ignore step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value arg_is_commissioning = False @@ -115,7 +120,7 @@ def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: ].value.value python_steps.append( - PythonTestStep(label=step_name, is_commissioning=arg_is_commissioning) + THTestStep(label=step_name, is_commissioning=arg_is_commissioning) ) return python_steps diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py index 8e30e22a..9fb9e551 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -22,7 +22,7 @@ ) from .python_test_folder import PythonTestFolder -from .python_test_models import PythonTest, PythonTestType +from .python_test_models import PythonTest, THTestType from .test_case import PythonTestCase from .test_suite import PythonTestSuite, SuiteType @@ -61,5 +61,5 @@ def __init__(self, test: PythonTest, python_test_version: str) -> None: ) @property - def test_type(self) -> PythonTestType: + def test_type(self) -> THTestType: return self.class_ref.python_test.type diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py index 03697248..6dffaf1d 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py @@ -25,8 +25,12 @@ TestCase, TestStep, ) +from test_collections.sdk_tests.support.models.th_test_models import ( + THTestStep, + THTestType, +) -from .yaml_test_models import YamlTest, YamlTestStep, YamlTestType +from .yaml_test_models import YamlTest # Custom type variable used to annotate the factory method in YamlTestCase. T = TypeVar("T", bound="YamlTestCase") @@ -76,11 +80,11 @@ async def setup(self) -> None: def class_factory(cls, test: YamlTest, yaml_version: str) -> Type[T]: """Dynamically declares a subclass based on the type of YAML test.""" case_class: Type[YamlTestCase] - if test.type == YamlTestType.MANUAL: + if test.type == THTestType.MANUAL: case_class = YamlManualTestCase - elif test.type == YamlTestType.SEMI_AUTOMATED: + elif test.type == THTestType.SEMI_AUTOMATED: case_class = YamlSemiAutomatedChipToolTestCase - elif test.type == YamlTestType.SIMULATED: + elif test.type == THTestType.SIMULATED: case_class = YamlSimulatedTestCase else: # Automated case_class = YamlChipToolTestCase @@ -143,7 +147,7 @@ def __title(cls, identifier: str, test_yaml: YamlTest) -> str: """ title = identifier - if test_yaml.type == YamlTestType.SEMI_AUTOMATED: + if test_yaml.type == THTestType.SEMI_AUTOMATED: title += " (Semi-automated)" if cls.__has_steps_disabled(test_yaml): @@ -151,7 +155,7 @@ def __title(cls, identifier: str, test_yaml: YamlTest) -> str: return title - def _append_automated_test_step(self, yaml_step: YamlTestStep) -> None: + def _append_automated_test_step(self, yaml_step: THTestStep) -> None: """ Disabled steps are ignored. (Such tests will be marked as 'Steps Disabled' elsewhere) diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py index e4ab474d..122e898a 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py @@ -20,11 +20,12 @@ TestCollectionDeclaration, TestSuiteDeclaration, ) +from test_collections.sdk_tests.support.models.th_test_models import THTestType from .test_case import YamlTestCase from .test_suite import SuiteType, YamlTestSuite from .yaml_test_folder import YamlTestFolder -from .yaml_test_models import YamlTest, YamlTestType +from .yaml_test_models import YamlTest class YamlCollectionDeclaration(TestCollectionDeclaration): @@ -59,5 +60,5 @@ def __init__(self, test: YamlTest, yaml_version: str) -> None: ) @property - def test_type(self) -> YamlTestType: + def test_type(self) -> THTestType: return self.class_ref.yaml_test.type diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py index ff59cff9..5828934d 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py @@ -13,38 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from enum import Enum -from pathlib import Path -from typing import Any, Optional +from typing import Any -from pydantic import BaseModel, Field from pydantic_yaml import YamlModelMixin +from ...models.th_test_models import THTest, THTestType + ### # This file declares YAML models that are used to parse the YAML Test Cases. ### -class YamlTestType(Enum): - AUTOMATED = 0 - SEMI_AUTOMATED = 1 - MANUAL = 2 - SIMULATED = 3 - - -class YamlTestStep(YamlModelMixin, BaseModel): - label: str - PICS: Optional[str] = None - verification: Optional[str] = None - command: Optional[str] - disabled: bool = False - arguments: Optional[dict[str, Any]] - - -class YamlTest(YamlModelMixin, BaseModel): - name: str - PICS: set[str] = set() - config: dict[str, Any] - steps: list[YamlTestStep] = Field(alias="tests") - type: YamlTestType = YamlTestType.MANUAL - path: Optional[Path] +class YamlTest(YamlModelMixin, THTest): + def __init__(self, **kwargs: Any) -> None: + super().__init__(steps=kwargs["tests"], **kwargs) + self.type: THTestType.MANUAL # type: ignore diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py index 007148ef..2602a2d3 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py @@ -18,14 +18,16 @@ from loguru import logger from pydantic import ValidationError -from .yaml_test_models import YamlTest, YamlTestType +from test_collections.sdk_tests.support.models.th_test_models import THTestType + +from .yaml_test_models import YamlTest class YamlParserException(Exception): """Raised when an error occurs during the parser of yaml file.""" -def _test_type(test: YamlTest) -> YamlTestType: +def _test_type(test: YamlTest) -> THTestType: """Determine the type of a test based on the parsed yaml. This is mainly determined by the number of disabled test steps. @@ -41,20 +43,20 @@ def _test_type(test: YamlTest) -> YamlTestType: - Simulated: Tests where file name have "Simulated" """ if test.path is not None and "Simulated" in str(test.path): - return YamlTestType.SIMULATED + return THTestType.SIMULATED steps = test.steps # If all disabled: if all(s.disabled is True for s in steps): - return YamlTestType.MANUAL + return THTestType.MANUAL # if any step has a UserPrompt, categorize as semi-automated if any(s.command == "UserPrompt" for s in steps): - return YamlTestType.SEMI_AUTOMATED + return THTestType.SEMI_AUTOMATED # Otherwise Automated - return YamlTestType.AUTOMATED + return THTestType.AUTOMATED def parse_yaml_test(path: Path) -> YamlTest: diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index b7f05dd7..4e1ac159 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -27,7 +27,7 @@ ) from .models.test_suite import SuiteType from .models.yaml_test_folder import YamlTestFolder -from .models.yaml_test_models import YamlTestType +from .models.yaml_test_models import THTestType from .models.yaml_test_parser import YamlParserException, parse_yaml_test ### @@ -94,9 +94,9 @@ def _parse_all_yaml( yaml_path=yaml_file, yaml_version=yaml_version ) - if test_case.test_type == YamlTestType.MANUAL: + if test_case.test_type == THTestType.MANUAL: suites[SuiteType.MANUAL].add_test_case(test_case) - elif test_case.test_type == YamlTestType.SIMULATED: + elif test_case.test_type == THTestType.SIMULATED: suites[SuiteType.SIMULATED].add_test_case(test_case) else: suites[SuiteType.AUTOMATED].add_test_case(test_case) From 8f5f77cda42c1639ce1000fad9bf0a464f51fe33 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Tue, 14 Nov 2023 17:07:32 +0000 Subject: [PATCH 22/45] Fix code violation --- app/tests/yaml_tests/test_sdk_yaml_collection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index f777d0fc..3c2049ee 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -18,7 +18,6 @@ import pytest -from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH from test_collections.sdk_tests.support.yaml_tests.models.test_declarations import ( YamlCaseDeclaration, YamlCollectionDeclaration, From e0dc0daeabacd8de242388072252e7da2b221b2e Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Thu, 16 Nov 2023 16:37:34 +0000 Subject: [PATCH 23/45] Code changes after review --- app/tests/python_tests/test_python_folder.py | 19 +++--- app/tests/python_tests/test_python_parser.py | 11 ++-- ...t_TC_ACL_1_1_Automated.py => TC_Sample.py} | 6 +- .../test_sdk_python_collection.py | 10 +--- .../yaml_tests/test_sdk_yaml_collection.py | 12 ++-- app/tests/yaml_tests/test_yaml_folder.py | 16 +++-- .../sdk_test_folder.py} | 20 ++++++- .../models/python_test_folder.py | 58 ------------------- .../models/python_test_models.py | 2 +- .../models/python_test_parser.py | 23 +++++--- .../models/test_declarations.py | 4 +- .../python_testing/sdk_python_test_tests.py | 6 +- .../yaml_tests/models/test_declarations.py | 4 +- .../support/yaml_tests/sdk_yaml_tests.py | 10 ++-- 14 files changed, 77 insertions(+), 124 deletions(-) rename app/tests/python_tests/test_python_script/{UnitTest_TC_ACL_1_1_Automated.py => TC_Sample.py} (95%) rename test_collections/sdk_tests/support/{yaml_tests/models/yaml_test_folder.py => models/sdk_test_folder.py} (74%) delete mode 100644 test_collections/sdk_tests/support/python_testing/models/python_test_folder.py diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py index db690688..68a289d6 100644 --- a/app/tests/python_tests/test_python_folder.py +++ b/app/tests/python_tests/test_python_folder.py @@ -18,9 +18,7 @@ from pathlib import Path from unittest import mock -from test_collections.sdk_tests.support.python_testing.models.python_test_folder import ( - PythonTestFolder, -) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder test_python_path = Path("/test/python") @@ -31,11 +29,10 @@ def test_python_folder_version() -> None: # We mock open to read version_file_content and Path exists to ignore that we're # testing with a fake path with mock.patch( - "test_collections.sdk_tests.support.python_testing.models." - "python_test_folder.open", + "test_collections.sdk_tests.support.models.sdk_test_folder.open", new=mock.mock_open(read_data=version_file_content), ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: - python_test_folder = PythonTestFolder(test_python_path) + python_test_folder = SDKTestFolder(test_python_path) assert python_test_folder.version == version_file_content @@ -43,22 +40,20 @@ def test_python_folder_version() -> None: def test_python_folder_version_missing() -> None: expected_version = "Unknown" with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: - python_folder = PythonTestFolder(test_python_path) + python_folder = SDKTestFolder(test_python_path) assert python_folder.version == expected_version def test_python_folder_filename_pattern() -> None: - """Test PythonTestFolder will search for files with filename pattern.""" + """Test SDKTestFolder will search for files with filename pattern.""" with mock.patch.object(target=Path, attribute="glob") as path_glob: # Default file_name_pattern: * - python_folder = PythonTestFolder(test_python_path) + python_folder = SDKTestFolder(test_python_path) _ = python_folder.python_file_paths() path_glob.assert_called_once_with("*.py") path_glob.reset_mock() pattern = "TC_*" - python_test_folder = PythonTestFolder( - test_python_path, filename_pattern=pattern - ) + python_test_folder = SDKTestFolder(test_python_path, filename_pattern=pattern) _ = python_test_folder.python_file_paths() path_glob.assert_called_once_with(f"{pattern}.py") diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py index 24167627..f0232b9e 100644 --- a/app/tests/python_tests/test_python_parser.py +++ b/app/tests/python_tests/test_python_parser.py @@ -34,8 +34,7 @@ def steps_TC_Sample(self) -> list[TestStep]: ] return steps - @async_test_body - async def test_steps_TC_Sample(self): + def test_steps_TC_Sample(self): print("Test execution") """ @@ -54,9 +53,11 @@ def steps_TC_Sample(self) -> list[TestStep]: ] return steps - @async_test_body - async def test_steps_TC_Sample(self): + def test_TC_Sample(self): print("Test execution") + + def pics_TC_Sample(self): + pics = ["MCORE.ROLE.COMMISSIONEE"] """ @@ -78,7 +79,7 @@ def test_python_file_parser_throws_pythonparserexception() -> None: def test_python_file_parser() -> None: - file_path = Path("/test/file.py") + file_path = Path("/test/TC_Sample.py") # We mock builtin `open` method to read sample python file content, # to avoid having to load a real file. diff --git a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py b/app/tests/python_tests/test_python_script/TC_Sample.py similarity index 95% rename from app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py rename to app/tests/python_tests/test_python_script/TC_Sample.py index fd8f1972..4c29514b 100644 --- a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py +++ b/app/tests/python_tests/test_python_script/TC_Sample.py @@ -48,6 +48,8 @@ def steps_TC_Sample(self) -> list[TestStep]: ] return steps - @async_test_body - async def test_steps_TC_Sample(self): + def test_TC_Sample(self): print("Test execution") + + def pics_TC_Sample(self): + pics = ["PICS"] diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index 2bb5b5d1..eee78a46 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -20,9 +20,7 @@ import pytest -from test_collections.sdk_tests.support.python_testing.models.python_test_folder import ( - PythonTestFolder, -) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( THTestType, ) @@ -39,12 +37,10 @@ def python_test_collection() -> PythonCollectionDeclaration: test_sdk_python_path = Path(__file__).parent / "test_python_script" with mock.patch.object(Path, "exists", return_value=True), mock.patch( - "test_collections.sdk_tests.support.python_testing.models.python_test_folder.open", + "test_collections.sdk_tests.support.models.sdk_test_folder.open", new=mock.mock_open(read_data="unit-test-python-version"), ): - folder = PythonTestFolder( - path=test_sdk_python_path, filename_pattern="UnitTest_TC_*" - ) + folder = SDKTestFolder(path=test_sdk_python_path, filename_pattern="TC_*") return sdk_python_test_collection(folder) diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index 3c2049ee..53379d14 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -18,16 +18,12 @@ import pytest +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder +from test_collections.sdk_tests.support.models.th_test_models import THTestType from test_collections.sdk_tests.support.yaml_tests.models.test_declarations import ( YamlCaseDeclaration, YamlCollectionDeclaration, ) -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder import ( - YamlTestFolder, -) -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_models import ( - THTestType, -) from test_collections.sdk_tests.support.yaml_tests.sdk_yaml_tests import ( sdk_yaml_test_collection, ) @@ -37,10 +33,10 @@ def yaml_collection() -> YamlCollectionDeclaration: test_sdk_yaml_path = Path(__file__).parent / "test_yamls" with mock.patch.object(Path, "exists", return_value=True), mock.patch( - "test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder.open", + "test_collections.sdk_tests.support.models.sdk_test_folder.open", new=mock.mock_open(read_data="unit-test-yaml-version"), ): - folder = YamlTestFolder( + folder = SDKTestFolder( path=test_sdk_yaml_path, filename_pattern="UnitTest_TC_*" ) return sdk_yaml_test_collection(folder) diff --git a/app/tests/yaml_tests/test_yaml_folder.py b/app/tests/yaml_tests/test_yaml_folder.py index 65a5012e..93a511d1 100644 --- a/app/tests/yaml_tests/test_yaml_folder.py +++ b/app/tests/yaml_tests/test_yaml_folder.py @@ -16,9 +16,7 @@ from pathlib import Path from unittest import mock -from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder import ( - YamlTestFolder, -) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder test_yaml_path = Path("/test/yaml") @@ -29,10 +27,10 @@ def test_yaml_folder_version() -> None: # We mock open to read version_file_content and Path exists to ignore that we're # testing with a fake path with mock.patch( - "test_collections.sdk_tests.support.yaml_tests.models.yaml_test_folder.open", + "test_collections.sdk_tests.support.models.sdk_test_folder.open", new=mock.mock_open(read_data=version_file_content), ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: - yaml_folder = YamlTestFolder(test_yaml_path) + yaml_folder = SDKTestFolder(test_yaml_path) assert yaml_folder.version == version_file_content @@ -40,20 +38,20 @@ def test_yaml_folder_version() -> None: def test_yaml_folder_version_missing() -> None: expected_version = "Unknown" with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: - yaml_folder = YamlTestFolder(test_yaml_path) + yaml_folder = SDKTestFolder(test_yaml_path) assert yaml_folder.version == expected_version def test_yaml_folder_filename_pattern() -> None: - """Test YamlTestFolder will search for files with filename pattern.""" + """Test SDKTestFolder will search for files with filename pattern.""" with mock.patch.object(target=Path, attribute="glob") as path_glob: # Default file_name_patter: * - yaml_folder = YamlTestFolder(test_yaml_path) + yaml_folder = SDKTestFolder(test_yaml_path) _ = yaml_folder.yaml_file_paths() path_glob.assert_called_once_with("*.y*ml") path_glob.reset_mock() pattern = "TC_*" - yaml_folder = YamlTestFolder(test_yaml_path, filename_pattern=pattern) + yaml_folder = SDKTestFolder(test_yaml_path, filename_pattern=pattern) _ = yaml_folder.yaml_file_paths() path_glob.assert_called_once_with(f"{pattern}.y*ml") diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py b/test_collections/sdk_tests/support/models/sdk_test_folder.py similarity index 74% rename from test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py rename to test_collections/sdk_tests/support/models/sdk_test_folder.py index a9aaa313..e6dc9b0f 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_folder.py +++ b/test_collections/sdk_tests/support/models/sdk_test_folder.py @@ -21,10 +21,10 @@ VERSION_FILE_FILENAME = ".version" -class YamlTestFolder: - """Representing a folder with Test YAML files. +class SDKTestFolder: + """Representing a folder with SDK Test files. - Note: YAML version is read from .version file in folder on init. + Note: SDK version is read from .version file in folder on init. """ def __init__(self, path: Path, filename_pattern: str = "*") -> None: @@ -56,3 +56,17 @@ def yaml_file_paths(self) -> list[Path]: list[Path]: list of paths to YAML test files. """ return list(self.path.glob(self.filename_pattern + ".y*ml")) + + def python_file_paths(self) -> list[Path]: + """Get list of paths to Python test files in folder. + + Filename filter can be applied if only some files should be selected. + Note: filter is without extension. Will search for .py files + + Args: + filename_pattern (str, optional): custom file filter. Defaults to "*". + + Returns: + list[Path]: list of paths to Python test files. + """ + return list(self.path.glob(self.filename_pattern + ".py")) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py b/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py deleted file mode 100644 index 92410538..00000000 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_folder.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from pathlib import Path - -from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH - -UNKNOWN_version = "Unknown" -VERSION_FILE_FILENAME = ".version" - - -class PythonTestFolder: - """Representing a folder with Test Python files. - - Note: Python version is read from .version file in folder on init. - """ - - def __init__(self, path: Path, filename_pattern: str = "*") -> None: - self.path = path - self.filename_pattern = filename_pattern - self.version = self.__version() - - def __version(self) -> str: - """Read version string from .version file in - /app/backend/test_collections/sdk_tests/sdk_checkout path.""" - version_file_path = SDK_CHECKOUT_PATH / VERSION_FILE_FILENAME - - if not version_file_path.exists(): - return UNKNOWN_version - else: - with open(version_file_path, "r") as file: - return file.read().rstrip() - - def python_file_paths(self) -> list[Path]: - """Get list of paths to Python test files in folder. - - Filename filter can be applied if only some files should be selected. - Note: filter is without extension. Will search for .py files - - Args: - filename_pattern (str, optional): custom file filter. Defaults to "*". - - Returns: - list[Path]: list of paths to Python test files. - """ - return list(self.path.glob(self.filename_pattern + ".py")) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index 6cddb0ff..78398f38 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -25,4 +25,4 @@ class PythonTest(THTest): def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.type: THTestType.MANUAL # type: ignore + self.type: THTestType.AUTOMATED # type: ignore diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 1503535d..5b5e2017 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -57,11 +57,10 @@ def parse_python_test(path: Path) -> PythonTest: This will also annotate parsed python test with it's path and test type. """ - tc_info = __extract_tcs_info(path) + tc_name = path.name.split(".")[0] + tc_info = __extract_tcs_info(path, tc_name) if not tc_info.desc or not tc_info.steps: - # The file name from path - tc_name = path.name.split(".")[0] raise PythonParserException( f"Test Case {tc_name} does not have methods desc_{tc_name} " f"or steps_{tc_name}" @@ -76,8 +75,8 @@ def parse_python_test(path: Path) -> PythonTest: return test -def __extract_tcs_info(path: Path) -> PythonTestInfo: - # Currently PICS and config is not configured in Python Testing +def __extract_tcs_info(path: Path, tc_name: str) -> PythonTestInfo: + # Currently config is not configured in Python Testing tc_pics: list = [] tc_config: dict = {} @@ -92,10 +91,12 @@ def __extract_tcs_info(path: Path) -> PythonTestInfo: for class_ in classes: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] for method in methods: - if "desc_" in method.name: + if "desc_" + tc_name in method.name: tc_desc = method.body[BODY_INDEX].value.value # type: ignore - elif "steps_" in method.name: + elif "steps_" + tc_name in method.name: tc_steps = __retrieve_steps(method) + elif "pics_" + tc_name in method.name: + tc_pics = __retrieve_pics(method) return PythonTestInfo( desc=tc_desc, @@ -124,3 +125,11 @@ def __retrieve_steps(method: ast.FunctionDef) -> List[THTestStep]: ) return python_steps + + +def __retrieve_pics(method: ast.FunctionDef) -> list: + python_steps: list = [] + for step in method.body[BODY_INDEX].value.elts: # type: ignore + python_steps.append(step.value) + + return python_steps diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py index 9fb9e551..eb9a1319 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -20,15 +20,15 @@ TestCollectionDeclaration, TestSuiteDeclaration, ) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder -from .python_test_folder import PythonTestFolder from .python_test_models import PythonTest, THTestType from .test_case import PythonTestCase from .test_suite import PythonTestSuite, SuiteType class PythonCollectionDeclaration(TestCollectionDeclaration): - def __init__(self, folder: PythonTestFolder, name: str) -> None: + def __init__(self, folder: SDKTestFolder, name: str) -> None: super().__init__(path=str(folder.path), name=name) self.python_test_version = folder.version diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py index 5945505d..2ecca9c0 100644 --- a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -17,9 +17,9 @@ from loguru import logger +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH -from .models.python_test_folder import PythonTestFolder from .models.python_test_parser import PythonParserException, parse_python_test from .models.test_declarations import ( PythonCaseDeclaration, @@ -39,7 +39,7 @@ ### SDK_PYTHON_TEST_PATH = SDK_CHECKOUT_PATH / Path("python_testing/scripts/sdk") -SDK_PYTHON_TEST_FOLDER = PythonTestFolder( +SDK_PYTHON_TEST_FOLDER = SDKTestFolder( path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*" ) @@ -90,7 +90,7 @@ def _parse_all_sdk_python_tests( def sdk_python_test_collection( - python_test_folder: PythonTestFolder = SDK_PYTHON_TEST_FOLDER, + python_test_folder: SDKTestFolder = SDK_PYTHON_TEST_FOLDER, ) -> PythonCollectionDeclaration: """Declare a new collection of test suites.""" collection = PythonCollectionDeclaration( diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py index 122e898a..63e81646 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py @@ -20,16 +20,16 @@ TestCollectionDeclaration, TestSuiteDeclaration, ) +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.models.th_test_models import THTestType from .test_case import YamlTestCase from .test_suite import SuiteType, YamlTestSuite -from .yaml_test_folder import YamlTestFolder from .yaml_test_models import YamlTest class YamlCollectionDeclaration(TestCollectionDeclaration): - def __init__(self, folder: YamlTestFolder, name: str) -> None: + def __init__(self, folder: SDKTestFolder, name: str) -> None: super().__init__(path=str(folder.path), name=name) self.yaml_version = folder.version diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index 4e1ac159..efefdca7 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -18,6 +18,7 @@ from loguru import logger +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH from .models.test_declarations import ( @@ -26,7 +27,6 @@ YamlSuiteDeclaration, ) from .models.test_suite import SuiteType -from .models.yaml_test_folder import YamlTestFolder from .models.yaml_test_models import THTestType from .models.yaml_test_parser import YamlParserException, parse_yaml_test @@ -43,10 +43,10 @@ YAML_PATH = SDK_CHECKOUT_PATH / "yaml_tests/yaml" SDK_YAML_PATH = YAML_PATH / "sdk" -SDK_YAML_TEST_FOLDER = YamlTestFolder(path=SDK_YAML_PATH, filename_pattern="Test_TC*") +SDK_YAML_TEST_FOLDER = SDKTestFolder(path=SDK_YAML_PATH, filename_pattern="Test_TC*") CUSTOM_YAML_PATH = YAML_PATH / "custom" -CUSTOM_YAML_TEST_FOLDER = YamlTestFolder( +CUSTOM_YAML_TEST_FOLDER = SDKTestFolder( path=CUSTOM_YAML_PATH, filename_pattern="Test_TC*" ) @@ -109,7 +109,7 @@ def _parse_all_yaml( def sdk_yaml_test_collection( - yaml_test_folder: YamlTestFolder = SDK_YAML_TEST_FOLDER, + yaml_test_folder: SDKTestFolder = SDK_YAML_TEST_FOLDER, ) -> YamlCollectionDeclaration: """Declare a new collection of test suites with the 3 test suites.""" collection = YamlCollectionDeclaration( @@ -128,7 +128,7 @@ def sdk_yaml_test_collection( def custom_yaml_test_collection( - yaml_test_folder: YamlTestFolder = CUSTOM_YAML_TEST_FOLDER, + yaml_test_folder: SDKTestFolder = CUSTOM_YAML_TEST_FOLDER, ) -> Optional[YamlCollectionDeclaration]: """Declare a new collection of test suites.""" collection = YamlCollectionDeclaration( From 352f80be959aef267fb4b7fafcc6014f88b55ba5 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 22 Nov 2023 12:30:48 +0000 Subject: [PATCH 24/45] Code changes after code review --- .version_information | 1 - app/chip_tool/chip_tool.py | 3 +- app/tests/python_tests/test_python_folder.py | 5 +- app/tests/python_tests/test_python_parser.py | 2 +- .../python_tests/test_python_test_case.py | 26 ++--- .../python_tests/test_python_test_suite.py | 7 +- .../test_sdk_python_collection.py | 6 +- .../yaml_tests/test_sdk_yaml_collection.py | 18 +-- app/tests/yaml_tests/test_test_case.py | 72 ++++++------ app/tests/yaml_tests/test_yaml_folder.py | 4 +- app/tests/yaml_tests/test_yaml_parser.py | 34 +++--- ...h_test_models.py => matter_test_models.py} | 10 +- .../support/models/sdk_test_folder.py | 25 +--- .../support/python_testing/models/__init__.py | 2 +- .../models/python_test_models.py | 6 +- .../models/python_test_parser.py | 110 ++++++++---------- .../python_testing/models/test_case.py | 6 +- .../models/test_declarations.py | 4 +- .../python_testing/models/test_suite.py | 10 +- .../python_testing/sdk_python_test_tests.py | 2 +- .../support/yaml_tests/models/test_case.py | 16 +-- .../yaml_tests/models/test_declarations.py | 4 +- .../yaml_tests/models/yaml_test_models.py | 6 +- .../yaml_tests/models/yaml_test_parser.py | 12 +- .../support/yaml_tests/sdk_yaml_tests.py | 15 ++- 25 files changed, 181 insertions(+), 225 deletions(-) delete mode 100644 .version_information rename test_collections/sdk_tests/support/models/{th_test_models.py => matter_test_models.py} (86%) diff --git a/.version_information b/.version_information deleted file mode 100644 index e6efee47..00000000 --- a/.version_information +++ /dev/null @@ -1 +0,0 @@ -main - dev diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 2b184091..f1bec756 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -132,7 +132,6 @@ class ChipToolUnknownTestType(Exception): class ChipToolTestType(str, Enum): CHIP_TOOL = "chip-tool" CHIP_APP = "chip-app" - PYTHON_TEST = "python-test" class ChipTool(metaclass=Singleton): @@ -588,7 +587,7 @@ async def run_test( if test_type == ChipToolTestType.CHIP_TOOL: test_path = f"{YAML_TESTS_PATH}/{test_id}.yaml" - elif test_type == ChipToolTestType.CHIP_APP: + else: test_path = f"{YAML_TESTS_PATH}/{test_id}_Simulated.yaml" parser_config = TestParserConfig(pics_path, self.specifications, test_options) diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py index 68a289d6..3636f8bf 100644 --- a/app/tests/python_tests/test_python_folder.py +++ b/app/tests/python_tests/test_python_folder.py @@ -14,7 +14,6 @@ # limitations under the License. # # flake8: noqa -# Ignore flake8 check for this file from pathlib import Path from unittest import mock @@ -49,11 +48,11 @@ def test_python_folder_filename_pattern() -> None: with mock.patch.object(target=Path, attribute="glob") as path_glob: # Default file_name_pattern: * python_folder = SDKTestFolder(test_python_path) - _ = python_folder.python_file_paths() + _ = python_folder.file_paths(extension=".py") path_glob.assert_called_once_with("*.py") path_glob.reset_mock() pattern = "TC_*" python_test_folder = SDKTestFolder(test_python_path, filename_pattern=pattern) - _ = python_test_folder.python_file_paths() + _ = python_test_folder.file_paths(extension=".py") path_glob.assert_called_once_with(f"{pattern}.py") diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py index f0232b9e..1eae8492 100644 --- a/app/tests/python_tests/test_python_parser.py +++ b/app/tests/python_tests/test_python_parser.py @@ -74,7 +74,7 @@ def test_python_file_parser_throws_pythonparserexception() -> None: parse_python_test(file_path) except PythonParserException as e: assert ( - "Test Case file does not have methods desc_file or steps_file" == str(e) + "/test/file.py must have a class name file" == str(e) ) diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index 98cd8b8a..4f62c7bc 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -25,9 +25,9 @@ from app.chip_tool.test_case import TestError from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger -from test_collections.sdk_tests.support.models.th_test_models import ( - THTestStep, - THTestType, +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, ) from test_collections.sdk_tests.support.python_testing.models import PythonTestCase from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( @@ -45,8 +45,8 @@ def python_test_instance( "param1": "value1", "param2": {"type": "config_type", "defaultValue": "value2"}, }, - steps: list[THTestStep] = [], - type: THTestType = THTestType.AUTOMATED, + steps: list[MatterTestStep] = [], + type: MatterTestType = MatterTestType.AUTOMATED, path: Optional[Path] = None, ) -> PythonTest: return PythonTest( @@ -129,7 +129,7 @@ def test_python_test_case_class_default_test_parameters() -> None: def test_automated_test_case_class_factory_subclass_mapping() -> None: """Test Automated tests are created as a subclass of PythonChipToolTestCase.""" - test = python_test_instance(type=THTestType.AUTOMATED) + test = python_test_instance(type=MatterTestType.AUTOMATED) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" ) @@ -178,13 +178,11 @@ def test_class_factory_test_class_name() -> None: def test_test_type_for_automated_tests() -> None: """Test that automated tests are set to use chip-tool""" - test = python_test_instance(type=THTestType.AUTOMATED) + test = python_test_instance(type=MatterTestType.AUTOMATED) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" ) assert issubclass(case_class, PythonChipToolTestCase) - instance = case_class(TestCaseExecution()) - assert instance.test_type == ChipToolTestType.PYTHON_TEST @pytest.mark.asyncio @@ -193,7 +191,7 @@ async def test_python_version_logging() -> None: Note that since `chip-tool` is not setup, we except the TestError raised. """ - for type in list(THTestType): + for type in list(MatterTestType): test = python_test_instance(type=type) test_python_version = "PythonVersionTest" case_class: Type[PythonTestCase] = PythonTestCase.class_factory( @@ -214,8 +212,8 @@ async def test_python_version_logging() -> None: def test_normal_steps_for_python_tests() -> None: """Test that python tests include enabled steps.""" - for type in list(THTestType): - test_step = THTestStep(label="Step1") + for type in list(MatterTestType): + test_step = MatterTestStep(label="Step1") test = python_test_instance(type=type, steps=[test_step]) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" @@ -228,8 +226,8 @@ def test_normal_steps_for_python_tests() -> None: def test_multiple_steps_for_python_tests() -> None: """Test that python tests multiple enabled steps are all included.""" - for type in list(THTestType): - test_step = THTestStep(label="StepN") + for type in list(MatterTestType): + test_step = MatterTestStep(label="StepN") no_steps = 5 test = python_test_instance(type=type, steps=([test_step] * no_steps)) case_class: Type[PythonTestCase] = PythonTestCase.class_factory( diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py index 28ed6631..5a230bab 100644 --- a/app/tests/python_tests/test_python_test_suite.py +++ b/app/tests/python_tests/test_python_test_suite.py @@ -66,8 +66,6 @@ def test_automated_suite_subclass() -> None: suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( suite_type=type, name="SomeSuite", python_test_version="some_version" ) - assert issubclass(suite_class, ChipToolPythonTestSuite) - assert suite_class.test_type == ChipToolTestType.PYTHON_TEST @pytest.mark.asyncio @@ -109,9 +107,6 @@ async def test_chip_tool_suite_setup() -> None: with mock.patch( "test_collections.sdk_tests.support.python_testing.models.test_suite.PythonTestSuite.setup" - ) as python_suite_setup, mock.patch( - "app.chip_tool.test_suite.ChipToolSuite.setup" - ) as chip_tool_suite_setup: + ) as python_suite_setup: await suite_instance.setup() python_suite_setup.assert_called_once() - chip_tool_suite_setup.assert_called_once() diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index eee78a46..3a5c6a68 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -22,7 +22,7 @@ from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( - THTestType, + MatterTestType, ) from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( PythonCaseDeclaration, @@ -68,9 +68,9 @@ def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> automated_suite = python_test_collection.test_suites["Python Testing Suite"] assert len(automated_suite.test_cases) == expected_automated_test_cases - type_count = dict.fromkeys(THTestType, 0) + type_count = dict.fromkeys(MatterTestType, 0) for test_case in automated_suite.test_cases.values(): assert isinstance(test_case, PythonCaseDeclaration) type_count[test_case.test_type] += 1 - assert type_count[THTestType.AUTOMATED] == expected_automated_test_cases + assert type_count[MatterTestType.AUTOMATED] == expected_automated_test_cases diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index 53379d14..71a88864 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -18,8 +18,8 @@ import pytest +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder -from test_collections.sdk_tests.support.models.th_test_models import THTestType from test_collections.sdk_tests.support.yaml_tests.models.test_declarations import ( YamlCaseDeclaration, YamlCollectionDeclaration, @@ -62,7 +62,7 @@ def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: assert len(manual_suite.test_cases) == expected_manual_test_cases for test_case in manual_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) - assert test_case.test_type == THTestType.MANUAL + assert test_case.test_type == MatterTestType.MANUAL def test_automated_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -79,15 +79,17 @@ def test_automated_suite(yaml_collection: YamlCollectionDeclaration) -> None: == expected_automated_test_cases + expected_semi_automated_test_cases ) - type_count = dict.fromkeys(THTestType, 0) + type_count = dict.fromkeys(MatterTestType, 0) for test_case in automated_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) type_count[test_case.test_type] += 1 - assert type_count[THTestType.AUTOMATED] == expected_automated_test_cases - assert type_count[THTestType.SEMI_AUTOMATED] == expected_semi_automated_test_cases - assert type_count[THTestType.SIMULATED] == expected_simulated_test_cases - assert type_count[THTestType.MANUAL] == expected_manual_test_cases + assert type_count[MatterTestType.AUTOMATED] == expected_automated_test_cases + assert ( + type_count[MatterTestType.SEMI_AUTOMATED] == expected_semi_automated_test_cases + ) + assert type_count[MatterTestType.SIMULATED] == expected_simulated_test_cases + assert type_count[MatterTestType.MANUAL] == expected_manual_test_cases def test_simulated_suite(yaml_collection: YamlCollectionDeclaration) -> None: @@ -98,4 +100,4 @@ def test_simulated_suite(yaml_collection: YamlCollectionDeclaration) -> None: assert len(simulated_suite.test_cases) == expected_simulated_test_cases for test_case in simulated_suite.test_cases.values(): assert isinstance(test_case, YamlCaseDeclaration) - assert test_case.test_type == THTestType.SIMULATED + assert test_case.test_type == MatterTestType.SIMULATED diff --git a/app/tests/yaml_tests/test_test_case.py b/app/tests/yaml_tests/test_test_case.py index 90a51900..9b77ec22 100644 --- a/app/tests/yaml_tests/test_test_case.py +++ b/app/tests/yaml_tests/test_test_case.py @@ -24,9 +24,9 @@ from app.models.test_case_execution import TestCaseExecution from app.test_engine.logger import test_engine_logger from app.test_engine.models.manual_test_case import ManualVerificationTestStep -from test_collections.sdk_tests.support.models.th_test_models import ( - THTestStep, - THTestType, +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, ) from test_collections.sdk_tests.support.yaml_tests.models import YamlTestCase from test_collections.sdk_tests.support.yaml_tests.models.test_case import ( @@ -47,8 +47,8 @@ def yaml_test_instance( "param1": "value1", "param2": {"type": "config_type", "defaultValue": "value2"}, }, - tests: list[THTestStep] = [], - type: THTestType = THTestType.AUTOMATED, + tests: list[MatterTestStep] = [], + type: MatterTestType = MatterTestType.AUTOMATED, path: Optional[Path] = None, ) -> YamlTest: return YamlTest( @@ -130,7 +130,7 @@ def test_yaml_test_case_class_default_test_parameters() -> None: def test_manual_test_case_class_factory_subclass_mapping() -> None: """Test Manual tests are created as a subclass of YamlManualTestCase.""" - test = yaml_test_instance(type=THTestType.MANUAL) + test = yaml_test_instance(type=MatterTestType.MANUAL) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -140,7 +140,7 @@ def test_manual_test_case_class_factory_subclass_mapping() -> None: def test_automated_test_case_class_factory_subclass_mapping() -> None: """Test Automated tests are created as a subclass of YamlChipToolTestCase.""" - test = yaml_test_instance(type=THTestType.AUTOMATED) + test = yaml_test_instance(type=MatterTestType.AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -150,7 +150,7 @@ def test_automated_test_case_class_factory_subclass_mapping() -> None: def test_semi_automated_test_case_class_factory_subclass_mapping() -> None: """Test Semi-Automated tests are created as a subclass of YamlSemiAutomatedChipToolTestCase.""" - test = yaml_test_instance(type=THTestType.SEMI_AUTOMATED) + test = yaml_test_instance(type=MatterTestType.SEMI_AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -160,7 +160,7 @@ def test_semi_automated_test_case_class_factory_subclass_mapping() -> None: def test_simulated_test_case_class_factory_subclass_mapping() -> None: """Test Simulated tests are created as a subclass of YamlSimulatedTestCase.""" - test = yaml_test_instance(type=THTestType.SIMULATED) + test = yaml_test_instance(type=MatterTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -170,7 +170,7 @@ def test_simulated_test_case_class_factory_subclass_mapping() -> None: def test_incomplete_test_case_class_factory_subclass_mapping() -> None: """Test Semi-Automated tests are created as a subclass of YamlSimulatedTestCase.""" - test = yaml_test_instance(type=THTestType.SIMULATED) + test = yaml_test_instance(type=MatterTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -221,13 +221,13 @@ def test_class_factory_test_title_semi_automated() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1', use it as metadata title, and append (Semi-automated) when appropriate.""" - for type in list(THTestType): + for type in list(MatterTestType): test = yaml_test_instance(type=type, name="TC-AB-1.2") case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) - if type == THTestType.SEMI_AUTOMATED: + if type == MatterTestType.SEMI_AUTOMATED: assert test.name in case_class.metadata["title"] assert "(Semi-automated)" in case_class.metadata["title"] else: @@ -237,12 +237,12 @@ def test_class_factory_test_title_semi_automated() -> None: def test_class_factory_test_title_steps_disabled() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1', use it as metadata title, and (Steps Disabled) when some but not all steps are disabled.""" - disabled_step = THTestStep(label="Step1", disabled=True) - enabled_step = THTestStep(label="Step2", disabled=False) + disabled_step = MatterTestStep(label="Step1", disabled=True) + enabled_step = MatterTestStep(label="Step2", disabled=False) for type in [ - THTestType.AUTOMATED, - THTestType.SEMI_AUTOMATED, - THTestType.SIMULATED, + MatterTestType.AUTOMATED, + MatterTestType.SEMI_AUTOMATED, + MatterTestType.SIMULATED, ]: test = yaml_test_instance( type=type, name="TC-AB-1.2", tests=[disabled_step, enabled_step] @@ -263,10 +263,10 @@ def test_steps_in_manual_yaml_test_case() -> None: - verification details are optional but passed when present in yaml """ steps = [ - THTestStep(label="Step1"), - THTestStep(label="Step2Verification", verification="Verification String"), + MatterTestStep(label="Step1"), + MatterTestStep(label="Step2Verification", verification="Verification String"), ] - test = yaml_test_instance(type=THTestType.MANUAL, tests=steps) + test = yaml_test_instance(type=MatterTestType.MANUAL, tests=steps) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -283,7 +283,7 @@ def test_steps_in_manual_yaml_test_case() -> None: def test_test_type_for_automated_tests() -> None: """Test that automated tests are set to use chip-tool""" - test = yaml_test_instance(type=THTestType.AUTOMATED) + test = yaml_test_instance(type=MatterTestType.AUTOMATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -294,7 +294,7 @@ def test_test_type_for_automated_tests() -> None: def test_test_type_for_simulated_tests() -> None: """Test that simulated tests are set to use chip-app""" - test = yaml_test_instance(type=THTestType.SIMULATED) + test = yaml_test_instance(type=MatterTestType.SIMULATED) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -309,7 +309,7 @@ async def test_yaml_version_logging() -> None: Note that since `chip-tool` is not setup, we except the TestError raised. """ - for type in list(THTestType): + for type in list(MatterTestType): test = yaml_test_instance(type=type) test_yaml_version = "YamlVersionTest" case_class: Type[YamlTestCase] = YamlTestCase.class_factory( @@ -329,7 +329,7 @@ async def test_yaml_version_logging() -> None: def test_default_first_steps_for_yaml_chip_tool_test_case() -> None: - test = yaml_test_instance(type=THTestType.AUTOMATED, tests=[]) + test = yaml_test_instance(type=MatterTestType.AUTOMATED, tests=[]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -339,7 +339,7 @@ def test_default_first_steps_for_yaml_chip_tool_test_case() -> None: def test_no_default_first_steps_for_yaml_simulated_test_case() -> None: - test = yaml_test_instance(type=THTestType.SIMULATED, tests=[]) + test = yaml_test_instance(type=MatterTestType.SIMULATED, tests=[]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) @@ -350,10 +350,10 @@ def test_no_default_first_steps_for_yaml_simulated_test_case() -> None: def test_disabled_steps_for_non_manual_test() -> None: """Test that non-manual tests skip disabled steps.""" - for type in list(THTestType): - if type == THTestType.MANUAL: + for type in list(MatterTestType): + if type == MatterTestType.MANUAL: continue - test_step = THTestStep(label="Disabled Step", disabled=True) + test_step = MatterTestStep(label="Disabled Step", disabled=True) test = yaml_test_instance(type=type, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" @@ -365,10 +365,10 @@ def test_disabled_steps_for_non_manual_test() -> None: def test_normal_steps_for_non_manual_tests() -> None: """Test that non-manual tests include enabled steps.""" - for type in list(THTestType): - if type == THTestType.MANUAL: + for type in list(MatterTestType): + if type == MatterTestType.MANUAL: continue - test_step = THTestStep(label="Step1") + test_step = MatterTestStep(label="Step1") test = yaml_test_instance(type=type, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" @@ -381,10 +381,10 @@ def test_normal_steps_for_non_manual_tests() -> None: def test_multiple_steps_for_non_manual() -> None: """Test that non-manual tests multiple enabled steps are all included.""" - for type in list(THTestType): - if type == THTestType.MANUAL: + for type in list(MatterTestType): + if type == MatterTestType.MANUAL: continue - test_step = THTestStep(label="StepN") + test_step = MatterTestStep(label="StepN") no_steps = 5 test = yaml_test_instance(type=type, tests=([test_step] * no_steps)) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( @@ -399,13 +399,13 @@ def test_multiple_steps_for_non_manual() -> None: def test_prompt_steps_for_yaml_chip_tool_test_case() -> None: - test_step = THTestStep( + test_step = MatterTestStep( label="Step1", command="UserPrompt", verification="Verify that This happened", ) - test = yaml_test_instance(type=THTestType.AUTOMATED, tests=[test_step]) + test = yaml_test_instance(type=MatterTestType.AUTOMATED, tests=[test_step]) case_class: Type[YamlTestCase] = YamlTestCase.class_factory( test=test, yaml_version="version" ) diff --git a/app/tests/yaml_tests/test_yaml_folder.py b/app/tests/yaml_tests/test_yaml_folder.py index 93a511d1..160c537e 100644 --- a/app/tests/yaml_tests/test_yaml_folder.py +++ b/app/tests/yaml_tests/test_yaml_folder.py @@ -47,11 +47,11 @@ def test_yaml_folder_filename_pattern() -> None: with mock.patch.object(target=Path, attribute="glob") as path_glob: # Default file_name_patter: * yaml_folder = SDKTestFolder(test_yaml_path) - _ = yaml_folder.yaml_file_paths() + _ = yaml_folder.file_paths(extension=".y*ml") path_glob.assert_called_once_with("*.y*ml") path_glob.reset_mock() pattern = "TC_*" yaml_folder = SDKTestFolder(test_yaml_path, filename_pattern=pattern) - _ = yaml_folder.yaml_file_paths() + _ = yaml_folder.file_paths(extension=".y*ml") path_glob.assert_called_once_with(f"{pattern}.y*ml") diff --git a/app/tests/yaml_tests/test_yaml_parser.py b/app/tests/yaml_tests/test_yaml_parser.py index 2c39938e..dcf040a9 100644 --- a/app/tests/yaml_tests/test_yaml_parser.py +++ b/app/tests/yaml_tests/test_yaml_parser.py @@ -20,9 +20,9 @@ from pydantic.error_wrappers import ValidationError from app.tests.yaml_tests.test_test_case import yaml_test_instance -from test_collections.sdk_tests.support.models.th_test_models import ( - THTestStep, - THTestType, +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, ) from test_collections.sdk_tests.support.yaml_tests.models.yaml_test_parser import ( YamlParserException, @@ -97,54 +97,54 @@ def test_yaml_file_parser() -> None: def test_test_type_all_disabled_steps() -> None: - disabled_step = THTestStep(label="Disabled Test Step", disabled=True) + disabled_step = MatterTestStep(label="Disabled Test Step", disabled=True) five_disabled_steps_test = yaml_test_instance(tests=[disabled_step] * 5) type = _test_type(five_disabled_steps_test) - assert type == THTestType.MANUAL + assert type == MatterTestType.MANUAL # simulated in path overrides test type to simulated five_disabled_steps_test.path = Path("TC_XX_Simulated.yaml") type = _test_type(five_disabled_steps_test) - assert type == THTestType.SIMULATED + assert type == MatterTestType.SIMULATED def test_test_type_some_disabled_steps() -> None: - disabled_step = THTestStep(label="Disabled Test Step", disabled=True) - enabled_step = THTestStep(label="Enabled Test Step", disabled=False) + disabled_step = MatterTestStep(label="Disabled Test Step", disabled=True) + enabled_step = MatterTestStep(label="Enabled Test Step", disabled=False) test = yaml_test_instance(tests=[disabled_step, enabled_step]) type = _test_type(test) - assert type == THTestType.AUTOMATED + assert type == MatterTestType.AUTOMATED # simulated in path overrides test type to simulated test.path = Path("TC_XX_Simulated.yaml") type = _test_type(test) - assert type == THTestType.SIMULATED + assert type == MatterTestType.SIMULATED def test_test_type_all_enabled_steps_no_prompts() -> None: - enabled_step = THTestStep(label="Enabled Test Step") + enabled_step = MatterTestStep(label="Enabled Test Step") five_enabled_steps_test = yaml_test_instance(tests=[enabled_step] * 5) type = _test_type(five_enabled_steps_test) - assert type == THTestType.AUTOMATED + assert type == MatterTestType.AUTOMATED # simulated in path overrides test type to simulated five_enabled_steps_test.path = Path("TC_XX_Simulated.yaml") type = _test_type(five_enabled_steps_test) - assert type == THTestType.SIMULATED + assert type == MatterTestType.SIMULATED def test_test_type_all_enabled_steps_some_prompts() -> None: - enabled_step = THTestStep(label="Enable Test Step") - prompt_step = THTestStep(label="Prompt Test Step", command="UserPrompt") + enabled_step = MatterTestStep(label="Enable Test Step") + prompt_step = MatterTestStep(label="Prompt Test Step", command="UserPrompt") test = yaml_test_instance(tests=[enabled_step, prompt_step]) type = _test_type(test) - assert type == THTestType.SEMI_AUTOMATED + assert type == MatterTestType.SEMI_AUTOMATED # simulated in path overrides test type to simulated test.path = Path("TC_XX_Simulated.yaml") type = _test_type(test) - assert type == THTestType.SIMULATED + assert type == MatterTestType.SIMULATED diff --git a/test_collections/sdk_tests/support/models/th_test_models.py b/test_collections/sdk_tests/support/models/matter_test_models.py similarity index 86% rename from test_collections/sdk_tests/support/models/th_test_models.py rename to test_collections/sdk_tests/support/models/matter_test_models.py index e4f6675f..b173e484 100644 --- a/test_collections/sdk_tests/support/models/th_test_models.py +++ b/test_collections/sdk_tests/support/models/matter_test_models.py @@ -24,14 +24,14 @@ ### -class THTestType(Enum): +class MatterTestType(Enum): AUTOMATED = 0 SEMI_AUTOMATED = 1 MANUAL = 2 SIMULATED = 3 -class THTestStep(BaseModel): +class MatterTestStep(BaseModel): label: str PICS: Optional[str] = None verification: Optional[str] = None @@ -40,10 +40,10 @@ class THTestStep(BaseModel): arguments: Optional[dict[str, Any]] -class THTest(BaseModel): +class MatterTest(BaseModel): name: str PICS: set[str] = set() config: dict[str, Any] - steps: list[THTestStep] - type: THTestType = THTestType.MANUAL + steps: list[MatterTestStep] + type: MatterTestType = MatterTestType.MANUAL path: Optional[Path] diff --git a/test_collections/sdk_tests/support/models/sdk_test_folder.py b/test_collections/sdk_tests/support/models/sdk_test_folder.py index e6dc9b0f..93b8e6a6 100644 --- a/test_collections/sdk_tests/support/models/sdk_test_folder.py +++ b/test_collections/sdk_tests/support/models/sdk_test_folder.py @@ -43,30 +43,15 @@ def __version(self) -> str: with open(version_file_path, "r") as file: return file.read().rstrip() - def yaml_file_paths(self) -> list[Path]: - """Get list of paths to yaml files in folder. + def file_paths(self, extension: str = "*.*") -> list[Path]: + """Get list of paths in folder. Filename filter can be applied if only some files should be selected. - Note: filter is without extension. Will search for .yml and .yaml files Args: - filename_pattern (str, optional): custom file filter. Defaults to "*". + extension (str): custom file extension filter. Defaults to "*.*". Returns: - list[Path]: list of paths to YAML test files. + list[Path]: list of paths to test files. """ - return list(self.path.glob(self.filename_pattern + ".y*ml")) - - def python_file_paths(self) -> list[Path]: - """Get list of paths to Python test files in folder. - - Filename filter can be applied if only some files should be selected. - Note: filter is without extension. Will search for .py files - - Args: - filename_pattern (str, optional): custom file filter. Defaults to "*". - - Returns: - list[Path]: list of paths to Python test files. - """ - return list(self.path.glob(self.filename_pattern + ".py")) + return list(self.path.glob(self.filename_pattern + extension)) diff --git a/test_collections/sdk_tests/support/python_testing/models/__init__.py b/test_collections/sdk_tests/support/python_testing/models/__init__.py index d15bf56b..05be8e81 100644 --- a/test_collections/sdk_tests/support/python_testing/models/__init__.py +++ b/test_collections/sdk_tests/support/python_testing/models/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. # from .test_case import PythonTestCase -from .test_suite import PythonTestSuite, SuiteType +# from .test_suite import PythonTestSuite, SuiteType diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index 78398f38..a8e8bc4d 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -15,14 +15,14 @@ # from typing import Any -from ...models.th_test_models import THTest, THTestType +from ...models.matter_test_models import MatterTest, MatterTestType ### # This file declares Python test models that are used to parse the Python Test Cases. ### -class PythonTest(THTest): +class PythonTest(MatterTest): def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.type: THTestType.AUTOMATED # type: ignore + self.type: MatterTestType.AUTOMATED # type: ignore diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 5b5e2017..3076a83e 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -15,11 +15,11 @@ # import ast from pathlib import Path -from typing import List +from typing import List, Optional -from test_collections.sdk_tests.support.models.th_test_models import ( - THTestStep, - THTestType, +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, ) from .python_test_models import PythonTest @@ -33,82 +33,66 @@ class PythonParserException(Exception): """Raised when an error occurs during the parser of python file.""" -class PythonTestInfo: - """This class stores all the information from a python test case that came from - python test script file.""" - - def __init__( - self, - desc: str, - pics: list, - config: dict, - steps: list[THTestStep], - type: THTestType, - ) -> None: - self.desc = desc - self.pics = pics - self.config = config - self.steps = steps - self.type = type - - def parse_python_test(path: Path) -> PythonTest: """Parse a single Python test file into PythonTest model. This will also annotate parsed python test with it's path and test type. """ - tc_name = path.name.split(".")[0] - tc_info = __extract_tcs_info(path, tc_name) - if not tc_info.desc or not tc_info.steps: - raise PythonParserException( - f"Test Case {tc_name} does not have methods desc_{tc_name} " - f"or steps_{tc_name}" - ) + return __parse_test_case_from_file(path) - test = PythonTest( - name=tc_info.desc, steps=tc_info.steps, config=tc_info.config, PICS=tc_info.pics - ) - test.path = path - test.type = tc_info.type - return test +def __parse_test_case_from_file(path: Path) -> PythonTest: + with open(path, "r") as python_file: + parsed_python_file = ast.parse(python_file.read()) + classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] + + tc_name = path.name.split(".")[0] + try: + class_ = next(c for c in classes if tc_name in c.name) + except StopIteration as si: # Raised when `next` doesn't find a matching method + raise PythonParserException(f"{path} must have a class name {tc_name}") from si + + return __parse_test_case_from_class(class_=class_, path=path, tc_name=tc_name) -def __extract_tcs_info(path: Path, tc_name: str) -> PythonTestInfo: +def __parse_test_case_from_class( + class_: ast.ClassDef, path: Path, tc_name: str +) -> PythonTest: # Currently config is not configured in Python Testing - tc_pics: list = [] tc_config: dict = {} - with open(path, "r") as python_file: - parsed_python_file = ast.parse(python_file.read()) - classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] + desc_method_name = "desc_" + tc_name + steps_method_name = "steps_" + tc_name + pics_method_name = "pics_" + tc_name - # Get TC description and TC steps from python test file - tc_desc: str = "" - tc_steps: List[THTestStep] = [] - - for class_ in classes: - methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] - for method in methods: - if "desc_" + tc_name in method.name: - tc_desc = method.body[BODY_INDEX].value.value # type: ignore - elif "steps_" + tc_name in method.name: - tc_steps = __retrieve_steps(method) - elif "pics_" + tc_name in method.name: - tc_pics = __retrieve_pics(method) - - return PythonTestInfo( - desc=tc_desc, - pics=tc_pics, - config=tc_config, + methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] + try: + desc_method = next(m for m in methods if desc_method_name in m.name) + tc_desc = desc_method.body[BODY_INDEX].value.value # type: ignore + + steps_method = next(m for m in methods if steps_method_name in m.name) + tc_steps = __retrieve_steps(steps_method) + + pics_method = next(m for m in methods if pics_method_name in m.name) + tc_pics = __retrieve_pics(pics_method) + except StopIteration as si: # Raised when `next` doesn't find a matching method + raise PythonParserException( + f"{path} did not contain valid definition for {tc_name}" + ) from si + + return PythonTest( + name=tc_desc, steps=tc_steps, - type=THTestType.AUTOMATED, + config=tc_config, + PICS=tc_pics, + path=path, + type=MatterTestType.AUTOMATED, ) -def __retrieve_steps(method: ast.FunctionDef) -> List[THTestStep]: - python_steps: List[THTestStep] = [] +def __retrieve_steps(method: ast.FunctionDef) -> List[MatterTestStep]: + python_steps: List[MatterTestStep] = [] for step in method.body[BODY_INDEX].value.elts: # type: ignore step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value arg_is_commissioning = False @@ -121,7 +105,7 @@ def __retrieve_steps(method: ast.FunctionDef) -> List[THTestStep]: ].value.value python_steps.append( - THTestStep(label=step_name, is_commissioning=arg_is_commissioning) + MatterTestStep(label=step_name, is_commissioning=arg_is_commissioning) ) return python_steps diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 0054838e..4253551a 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -265,10 +265,8 @@ async def cleanup(self) -> None: self.chip_tool.destroy_device() -class PythonChipToolTestCase(PythonTestCase, ChipToolTest): - """Automated Python test cases using chip-tool.""" - - test_type = ChipToolTestType.PYTHON_TEST +class PythonChipToolTestCase(PythonTestCase): + """Automated Python test cases""" def create_test_steps(self) -> None: for step in self.python_test.steps: diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py index eb9a1319..a5936f2f 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -22,7 +22,7 @@ ) from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder -from .python_test_models import PythonTest, THTestType +from .python_test_models import PythonTest, MatterTestType from .test_case import PythonTestCase from .test_suite import PythonTestSuite, SuiteType @@ -61,5 +61,5 @@ def __init__(self, test: PythonTest, python_test_version: str) -> None: ) @property - def test_type(self) -> THTestType: + def test_type(self) -> MatterTestType: return self.class_ref.python_test.type diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 5b13e4a6..317e6fc1 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -16,16 +16,10 @@ from enum import Enum from typing import Type, TypeVar -from app.chip_tool.chip_tool import ChipToolTestType -from app.chip_tool.test_suite import ChipToolSuite from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestSuite -class PythonTestSuiteFactoryError(Exception): - pass - - class SuiteType(Enum): AUTOMATED = 1 @@ -79,10 +73,8 @@ def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: ) -class ChipToolPythonTestSuite(PythonTestSuite, ChipToolSuite): - test_type = ChipToolTestType.PYTHON_TEST +class ChipToolPythonTestSuite(PythonTestSuite): async def setup(self) -> None: """Due top multi inheritance, we need to call setup on both super classes.""" await PythonTestSuite.setup(self) - await ChipToolSuite.setup(self) diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py index 2ecca9c0..b31a04aa 100644 --- a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -97,7 +97,7 @@ def sdk_python_test_collection( name="SDK Python Tests", folder=python_test_folder ) - files = python_test_folder.python_file_paths() + files = python_test_folder.file_paths(extension=".py") version = python_test_folder.version suites = _parse_all_sdk_python_tests( python_test_files=files, python_test_version=version diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py index 6dffaf1d..b0ddf9a2 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_case.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_case.py @@ -25,9 +25,9 @@ TestCase, TestStep, ) -from test_collections.sdk_tests.support.models.th_test_models import ( - THTestStep, - THTestType, +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, ) from .yaml_test_models import YamlTest @@ -80,11 +80,11 @@ async def setup(self) -> None: def class_factory(cls, test: YamlTest, yaml_version: str) -> Type[T]: """Dynamically declares a subclass based on the type of YAML test.""" case_class: Type[YamlTestCase] - if test.type == THTestType.MANUAL: + if test.type == MatterTestType.MANUAL: case_class = YamlManualTestCase - elif test.type == THTestType.SEMI_AUTOMATED: + elif test.type == MatterTestType.SEMI_AUTOMATED: case_class = YamlSemiAutomatedChipToolTestCase - elif test.type == THTestType.SIMULATED: + elif test.type == MatterTestType.SIMULATED: case_class = YamlSimulatedTestCase else: # Automated case_class = YamlChipToolTestCase @@ -147,7 +147,7 @@ def __title(cls, identifier: str, test_yaml: YamlTest) -> str: """ title = identifier - if test_yaml.type == THTestType.SEMI_AUTOMATED: + if test_yaml.type == MatterTestType.SEMI_AUTOMATED: title += " (Semi-automated)" if cls.__has_steps_disabled(test_yaml): @@ -155,7 +155,7 @@ def __title(cls, identifier: str, test_yaml: YamlTest) -> str: return title - def _append_automated_test_step(self, yaml_step: THTestStep) -> None: + def _append_automated_test_step(self, yaml_step: MatterTestStep) -> None: """ Disabled steps are ignored. (Such tests will be marked as 'Steps Disabled' elsewhere) diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py index 63e81646..fa9bea11 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py @@ -21,7 +21,7 @@ TestSuiteDeclaration, ) from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder -from test_collections.sdk_tests.support.models.th_test_models import THTestType +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType from .test_case import YamlTestCase from .test_suite import SuiteType, YamlTestSuite @@ -60,5 +60,5 @@ def __init__(self, test: YamlTest, yaml_version: str) -> None: ) @property - def test_type(self) -> THTestType: + def test_type(self) -> MatterTestType: return self.class_ref.yaml_test.type diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py index 5828934d..be5208d2 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py @@ -17,14 +17,14 @@ from pydantic_yaml import YamlModelMixin -from ...models.th_test_models import THTest, THTestType +from ...models.matter_test_models import MatterTest, MatterTestType ### # This file declares YAML models that are used to parse the YAML Test Cases. ### -class YamlTest(YamlModelMixin, THTest): +class YamlTest(YamlModelMixin, MatterTest): def __init__(self, **kwargs: Any) -> None: super().__init__(steps=kwargs["tests"], **kwargs) - self.type: THTestType.MANUAL # type: ignore + self.type: MatterTestType.MANUAL # type: ignore diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py index 2602a2d3..f4275660 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_parser.py @@ -18,7 +18,7 @@ from loguru import logger from pydantic import ValidationError -from test_collections.sdk_tests.support.models.th_test_models import THTestType +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType from .yaml_test_models import YamlTest @@ -27,7 +27,7 @@ class YamlParserException(Exception): """Raised when an error occurs during the parser of yaml file.""" -def _test_type(test: YamlTest) -> THTestType: +def _test_type(test: YamlTest) -> MatterTestType: """Determine the type of a test based on the parsed yaml. This is mainly determined by the number of disabled test steps. @@ -43,20 +43,20 @@ def _test_type(test: YamlTest) -> THTestType: - Simulated: Tests where file name have "Simulated" """ if test.path is not None and "Simulated" in str(test.path): - return THTestType.SIMULATED + return MatterTestType.SIMULATED steps = test.steps # If all disabled: if all(s.disabled is True for s in steps): - return THTestType.MANUAL + return MatterTestType.MANUAL # if any step has a UserPrompt, categorize as semi-automated if any(s.command == "UserPrompt" for s in steps): - return THTestType.SEMI_AUTOMATED + return MatterTestType.SEMI_AUTOMATED # Otherwise Automated - return THTestType.AUTOMATED + return MatterTestType.AUTOMATED def parse_yaml_test(path: Path) -> YamlTest: diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index efefdca7..2d4eb797 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -26,8 +26,13 @@ YamlCollectionDeclaration, YamlSuiteDeclaration, ) + +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, +) + from .models.test_suite import SuiteType -from .models.yaml_test_models import THTestType from .models.yaml_test_parser import YamlParserException, parse_yaml_test ### @@ -94,9 +99,9 @@ def _parse_all_yaml( yaml_path=yaml_file, yaml_version=yaml_version ) - if test_case.test_type == THTestType.MANUAL: + if test_case.test_type == MatterTestType.MANUAL: suites[SuiteType.MANUAL].add_test_case(test_case) - elif test_case.test_type == THTestType.SIMULATED: + elif test_case.test_type == MatterTestType.SIMULATED: suites[SuiteType.SIMULATED].add_test_case(test_case) else: suites[SuiteType.AUTOMATED].add_test_case(test_case) @@ -116,7 +121,7 @@ def sdk_yaml_test_collection( name="SDK YAML Tests", folder=yaml_test_folder ) - files = yaml_test_folder.yaml_file_paths() + files = yaml_test_folder.file_paths(extension=".y*ml") version = yaml_test_folder.version suites = _parse_all_yaml(yaml_files=files, yaml_version=version) @@ -135,7 +140,7 @@ def custom_yaml_test_collection( name="Custom YAML Tests", folder=yaml_test_folder ) - files = yaml_test_folder.yaml_file_paths() + files = yaml_test_folder.file_paths(extension=".y*ml") suites = _parse_all_yaml(yaml_files=files, yaml_version="custom") for suite in suites: From 7f34cbfe1009b12898a8986e6e7cd5680117ff59 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 22 Nov 2023 12:33:34 +0000 Subject: [PATCH 25/45] Putting back .version_information --- .version_information | 1 + 1 file changed, 1 insertion(+) create mode 100644 .version_information diff --git a/.version_information b/.version_information new file mode 100644 index 00000000..a7118dce --- /dev/null +++ b/.version_information @@ -0,0 +1 @@ +main - dev \ No newline at end of file From 1502f2e931a1a058d03440044a3dd97f26d5f908 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 22 Nov 2023 12:34:36 +0000 Subject: [PATCH 26/45] Putting back .version_information --- .version_information | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version_information b/.version_information index a7118dce..e6efee47 100644 --- a/.version_information +++ b/.version_information @@ -1 +1 @@ -main - dev \ No newline at end of file +main - dev From 10ac46b7a597f61160608041bcceb48004af14cb Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 22 Nov 2023 12:38:55 +0000 Subject: [PATCH 27/45] Fixed mypy violations --- app/tests/python_tests/test_python_parser.py | 4 +--- .../support/python_testing/models/__init__.py | 2 +- .../support/python_testing/models/test_declarations.py | 2 +- .../support/python_testing/models/test_suite.py | 1 - .../support/yaml_tests/models/test_declarations.py | 2 +- .../sdk_tests/support/yaml_tests/sdk_yaml_tests.py | 10 ++++------ 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py index 1eae8492..c4eb37c2 100644 --- a/app/tests/python_tests/test_python_parser.py +++ b/app/tests/python_tests/test_python_parser.py @@ -73,9 +73,7 @@ def test_python_file_parser_throws_pythonparserexception() -> None: try: parse_python_test(file_path) except PythonParserException as e: - assert ( - "/test/file.py must have a class name file" == str(e) - ) + assert "/test/file.py must have a class name file" == str(e) def test_python_file_parser() -> None: diff --git a/test_collections/sdk_tests/support/python_testing/models/__init__.py b/test_collections/sdk_tests/support/python_testing/models/__init__.py index 05be8e81..d15bf56b 100644 --- a/test_collections/sdk_tests/support/python_testing/models/__init__.py +++ b/test_collections/sdk_tests/support/python_testing/models/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. # from .test_case import PythonTestCase -# from .test_suite import PythonTestSuite, SuiteType +from .test_suite import PythonTestSuite, SuiteType diff --git a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py index a5936f2f..62e8560a 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_declarations.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_declarations.py @@ -22,7 +22,7 @@ ) from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder -from .python_test_models import PythonTest, MatterTestType +from .python_test_models import MatterTestType, PythonTest from .test_case import PythonTestCase from .test_suite import PythonTestSuite, SuiteType diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 317e6fc1..f2c25b95 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -74,7 +74,6 @@ def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: class ChipToolPythonTestSuite(PythonTestSuite): - async def setup(self) -> None: """Due top multi inheritance, we need to call setup on both super classes.""" await PythonTestSuite.setup(self) diff --git a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py index fa9bea11..fbe36139 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/test_declarations.py @@ -20,8 +20,8 @@ TestCollectionDeclaration, TestSuiteDeclaration, ) -from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType +from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from .test_case import YamlTestCase from .test_suite import SuiteType, YamlTestSuite diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index 2d4eb797..e6d3fc83 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -18,6 +18,10 @@ from loguru import logger +from test_collections.sdk_tests.support.models.matter_test_models import ( + MatterTestStep, + MatterTestType, +) from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH @@ -26,12 +30,6 @@ YamlCollectionDeclaration, YamlSuiteDeclaration, ) - -from test_collections.sdk_tests.support.models.matter_test_models import ( - MatterTestStep, - MatterTestType, -) - from .models.test_suite import SuiteType from .models.yaml_test_parser import YamlParserException, parse_yaml_test From 17ec11af9be68993733b97726772d17b3e2215b6 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 22 Nov 2023 12:41:48 +0000 Subject: [PATCH 28/45] Removed unused imports --- .../support/python_testing/models/python_test_parser.py | 2 +- .../sdk_tests/support/python_testing/models/test_case.py | 3 +-- .../sdk_tests/support/yaml_tests/sdk_yaml_tests.py | 5 +---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 3076a83e..3c048492 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -15,7 +15,7 @@ # import ast from pathlib import Path -from typing import List, Optional +from typing import List from test_collections.sdk_tests.support.models.matter_test_models import ( MatterTestStep, diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 4253551a..1fba63a6 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -21,8 +21,7 @@ from matter_chip_tool_adapter.decoder import MatterLog -from app.chip_tool.chip_tool import ChipTool, ChipToolTestType -from app.chip_tool.test_case import ChipToolTest +from app.chip_tool.chip_tool import ChipTool from app.models import TestCaseExecution from app.test_engine.logger import ( CHIP_LOG_FORMAT, diff --git a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py index e6d3fc83..ff751996 100644 --- a/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py +++ b/test_collections/sdk_tests/support/yaml_tests/sdk_yaml_tests.py @@ -18,10 +18,7 @@ from loguru import logger -from test_collections.sdk_tests.support.models.matter_test_models import ( - MatterTestStep, - MatterTestType, -) +from test_collections.sdk_tests.support.models.matter_test_models import MatterTestType from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH From 77cf5746cc20b7641e7c62cc27c3c78d10cd0c90 Mon Sep 17 00:00:00 2001 From: Gladyston Franca <gladyston@apple.com> Date: Mon, 27 Nov 2023 03:42:00 -0300 Subject: [PATCH 29/45] adding rpc client code and mounting new volumes on sdk container --- app/chip_tool/chip_tool.py | 24 +++++-- app/chip_tool/test_suite.py | 3 +- app/tests/chip_tool/test_chip_tool.py | 35 ++++----- .../models/rpc_client/test_harness_client.py | 71 +++++++++++++++++++ .../python_testing/models/test_case.py | 37 +++++----- 5 files changed, 128 insertions(+), 42 deletions(-) create mode 100644 test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index f1bec756..dbff77d6 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -107,6 +107,17 @@ XML_SPEC_DEFINITION_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path( "sdk_runner/specifications/chip/" ) +# Python Testing Folder +# LOCAL_PYTHON_TESTING_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path( +# "python_testing/scripts/sdk" +# ) +# DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" + +# RPC Client Running on SDK Container +LOCAL_RPC_PYTHON_TESTING_PATH = BACKEND_ROOT / Path( + "test_collections/sdk_tests/support/python_testing/models/rpc_client/" +) +DOCKER_RPC_PYTHON_TESTING_PATH = "/root/python_testing/rpc_client" # Docker Network @@ -132,6 +143,7 @@ class ChipToolUnknownTestType(Exception): class ChipToolTestType(str, Enum): CHIP_TOOL = "chip-tool" CHIP_APP = "chip-app" + PYTHON_TEST = "python-test" class ChipTool(metaclass=Singleton): @@ -168,6 +180,10 @@ class ChipTool(metaclass=Singleton): "bind": DOCKER_CREDENTIALS_DEVELOPMENT_PATH, "mode": "ro", }, + LOCAL_RPC_PYTHON_TESTING_PATH: { + "bind": DOCKER_RPC_PYTHON_TESTING_PATH, + "mode": "rw", + }, }, } @@ -345,7 +361,7 @@ def __get_gateway_ip(self) -> str: .get(DOCKER_GATEWAY_KEY, "") ) - async def start_container_no_server(self) -> None: + async def start_container(self) -> None: """ Creates the chip-tool container without any server running (ChipTool or ChipApp). @@ -371,7 +387,7 @@ async def start_container_no_server(self) -> None: # Server started is false after spinning up a new container. self.__server_started = False - async def start_container( + async def start_server( self, test_type: ChipToolTestType, use_paa_certs: bool = False ) -> None: """Creates the chip-tool container. @@ -384,7 +400,7 @@ async def start_container( ) return - await self.start_container_no_server() + await self.start_container() web_socket_config = WebSocketRunnerConfig() web_socket_config.server_address = self.__get_gateway_ip() @@ -587,7 +603,7 @@ async def run_test( if test_type == ChipToolTestType.CHIP_TOOL: test_path = f"{YAML_TESTS_PATH}/{test_id}.yaml" - else: + elif test_type == ChipToolTestType.CHIP_APP: test_path = f"{YAML_TESTS_PATH}/{test_id}_Simulated.yaml" parser_config = TestParserConfig(pics_path, self.specifications, test_options) diff --git a/app/chip_tool/test_suite.py b/app/chip_tool/test_suite.py index 7fb57a85..c99af552 100644 --- a/app/chip_tool/test_suite.py +++ b/app/chip_tool/test_suite.py @@ -54,7 +54,7 @@ async def setup(self) -> None: logger.info("Setting up chip_tool") # Use test engine logger to log all events to test run. self.chip_tool.logger = logger - await self.chip_tool.start_container( + await self.chip_tool.start_server( self.test_type, self.config.dut_config.chip_tool_use_paa_certs ) @@ -284,3 +284,4 @@ async def __prompt_user_to_perform_decommission(self) -> None: f"Received unknown prompt option for \ decommissioning step: {prompt_response.response}" ) + diff --git a/app/tests/chip_tool/test_chip_tool.py b/app/tests/chip_tool/test_chip_tool.py index 139e8899..461bfbd2 100644 --- a/app/tests/chip_tool/test_chip_tool.py +++ b/app/tests/chip_tool/test_chip_tool.py @@ -69,7 +69,7 @@ async def test_start_container() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) mock_create_container.assert_called_once_with(docker_image, ChipTool.run_parameters) mock_start_chip_server.assert_awaited_once_with(test_type, False) @@ -98,7 +98,7 @@ async def test_start_container_using_paa_certs() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_container(test_type, use_paa_certs=True) + await chip_tool.start_server(test_type, use_paa_certs=True) mock_create_container.assert_called_once_with(docker_image, ChipTool.run_parameters) mock_start_chip_server.assert_awaited_once_with(test_type, True) @@ -120,7 +120,7 @@ async def test_not_start_container_when_running() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) mock_create_container.assert_not_called() mock_start_chip_server.assert_not_called() @@ -359,8 +359,8 @@ async def test_destroy_container_running() -> None: ), mock.patch.object( target=chip_tool, attribute="start_chip_server" ): - await chip_tool.start_container(test_type) - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) + await chip_tool.start_server(test_type) assert chip_tool._ChipTool__chip_tool_container is not None @@ -403,7 +403,7 @@ async def test_destroy_container_once() -> None: ), mock.patch.object( target=chip_tool, attribute="start_chip_server" ): - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) chip_tool.destroy_device() chip_tool.destroy_device() @@ -452,7 +452,7 @@ async def test_set_pics() -> None: target="app.chip_tool.chip_tool.subprocess.run", return_value=CompletedProcess(expected_command, 0), ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) chip_tool.set_pics(pics) @@ -505,7 +505,7 @@ async def test_send_command_default_prefix() -> None: target="app.chip_tool.chip_tool.exec_run_in_container", return_value=mock_result, ) as mock_exec_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) result = chip_tool.send_command(cmd, prefix=chip_tool_prefix) @@ -548,7 +548,7 @@ async def test_send_command_custom_prefix() -> None: target="app.chip_tool.chip_tool.exec_run_in_container", return_value=mock_result, ) as mock_exec_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) result = chip_tool.send_command(cmd, prefix=chip_tool_prefix) @@ -589,7 +589,7 @@ async def test_run_test_default_config() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -643,7 +643,7 @@ async def test_run_test_custom_timeout() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -691,7 +691,7 @@ async def test_run_test_with_custom_parameter() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -740,7 +740,7 @@ async def test_run_test_with_endpoint_parameter() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -788,7 +788,7 @@ async def test_run_test_with_nodeID_and_cluster_parameters() -> None: target="app.chip_tool.chip_tool.WebSocketRunner.run", return_value=True, ) as mock_run: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) await chip_tool.run_test( test_step_interface=TestRunnerHooks(), @@ -847,7 +847,7 @@ async def test_pairing_on_network_command_params() -> None: attribute="send_websocket_command", return_value='{"results": []}', ) as mock_send_websocket_command: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) # Send on-network pairing command result = await chip_tool.pairing_on_network( @@ -898,7 +898,7 @@ async def test_pairing_ble_wifi_command_params() -> None: attribute="send_websocket_command", return_value='{"results": []}', ) as mock_send_websocket_command: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) # Send BLE-WIFI pairing command result = await chip_tool.pairing_ble_wifi( @@ -953,7 +953,7 @@ async def test_pairing_ble_thread_command_params() -> None: return_value='{"results": []}', # '{ "results": [{ "error": "FAILURE" }] ) as mock_send_websocket_command: - await chip_tool.start_container(test_type) + await chip_tool.start_server(test_type) # Send BLE-THREAD pairing command result = await chip_tool.pairing_ble_thread( @@ -973,3 +973,4 @@ async def test_pairing_ble_thread_command_params() -> None: # clean up: chip_tool._ChipTool__chip_tool_container = None settings.CHIP_TOOL_TRACE = original_trace_setting_value + diff --git a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py new file mode 100644 index 00000000..415083cf --- /dev/null +++ b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib +import sys +from multiprocessing.managers import BaseManager + +from matter_testing_support import MatterTestConfig, run_tests + +try: + from matter_yamltests.hooks import TestRunnerHooks +except: + + class TestRunnerHooks: + pass + + +MATTER_DEVELOPMENT_PAA_ROOT_CERTS = "/paa-root-certs" + +# Pre-computed param list for each Python Test as defined in Verification Steps. +test_params = { + "TC_ACE_1_3": MatterTestConfig( + tests=["test_TC_ACE_1_3"], + commissioning_method="on-network", + discriminators=[3840], + setup_passcodes=[20202021], + dut_node_ids=[0x12344321], + paa_trust_store_path=MATTER_DEVELOPMENT_PAA_ROOT_CERTS, + storage_path="/root/admin_storage.json", + ) +} + + +def main(): + if len(sys.argv) != 2: + raise Exception("Python test id should be provided as the only parameter.") + + test_name = sys.argv[1] + + config = test_params.get(test_name) + + if config is None: + raise ValueError(f"Not a valid test id: {test_name}") + + module = importlib.import_module(test_name) + TestClassReference = getattr(module, test_name) + + BaseManager.register(TestRunnerHooks.__name__) + manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") + manager.connect() + test_runner_hooks = manager.TestRunnerHooks() # shared object proxy + + run_tests(TestClassReference, config, test_runner_hooks) + + +if __name__ == "__main__": + main() diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 1fba63a6..1ae9d886 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -38,8 +38,8 @@ T = TypeVar("T", bound="PythonTestCase") # Command line params -RUNNER_CLASS = "external_runner.py" -RUNNER_CLASS_PATH = "./testing2/" +RUNNER_CLASS = "test_harness_client.py" +RUNNER_CLASS_PATH = "/root/python_testing/rpc_client" EXECUTABLE = "python3" @@ -161,7 +161,7 @@ async def setup(self) -> None: try: await super().setup() self.chip_tool = ChipTool() - await self.chip_tool.start_container_no_server() + await self.chip_tool.start_container() assert self.chip_tool.is_running() except NotImplementedError: pass @@ -225,7 +225,7 @@ async def execute(self) -> None: manager.start() test_runner_hooks = manager.TestRunnerHooks() # type: ignore runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS - # TODO Ignoring stream from docker execution until client impl is done + # TODO Ignoring stream from docker execution self.chip_tool.send_command( f"{runner_class} {self.metadata['title']}", prefix=EXECUTABLE, @@ -239,26 +239,23 @@ async def execute(self) -> None: if not update: continue - def handle_update(update: dict) -> None: - def call_function(obj, func_name, kwargs) -> None: # type: ignore - func = getattr(obj, func_name, None) - if not func: - raise AttributeError( - f"{func_name} is not a method of {obj}" - ) - if not callable(func): - raise TypeError(f"{func_name} is not callable") - # Call the method with the unpacked keyword arguments. - func(**kwargs) - - for func_name, kwargs in update.items(): - call_function(self, func_name, kwargs) - - handle_update(update) + self.__handle_update(update) await sleep(0.1) finally: pass + def __handle_update(self, update: dict) -> None: + for func_name, kwargs in update.items(): + self.__call_function_from_name(self, func_name, kwargs) + + def __call_function_from_name(self, obj, func_name, kwargs) -> None: # type: ignore + func = getattr(obj, func_name, None) + if not func: + raise AttributeError(f"{func_name} is not a method of {obj}") + if not callable(func): + raise TypeError(f"{func_name} is not callable") + func(**kwargs) + async def cleanup(self) -> None: logger.info("Test Cleanup") self.chip_tool.destroy_device() From 2135ffdd160c4227e24611ea69b20660f320bc14 Mon Sep 17 00:00:00 2001 From: Gladyston Franca <gladyston@apple.com> Date: Mon, 27 Nov 2023 14:45:08 -0300 Subject: [PATCH 30/45] Fixing sdk rpc file path --- app/chip_tool/chip_tool.py | 6 +++--- .../python_testing/models/python_testing_hooks_proxy.py | 6 +++--- .../sdk_tests/support/python_testing/models/test_case.py | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 37746ae1..d2198eb4 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -114,10 +114,10 @@ # DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" # RPC Client Running on SDK Container -LOCAL_RPC_PYTHON_TESTING_PATH = BACKEND_ROOT / Path( - "test_collections/sdk_tests/support/python_testing/models/rpc_client/" +LOCAL_RPC_PYTHON_TESTING_PATH = Path( + "/home/ubuntu/certification-tool/backend/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py" ) -DOCKER_RPC_PYTHON_TESTING_PATH = "/root/python_testing/rpc_client" +DOCKER_RPC_PYTHON_TESTING_PATH = "/root/python_testing/test_harness_client.py" # Docker Network diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index c26078da..5dce6255 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -20,7 +20,7 @@ class SDKPythonTestRunnerHooks(TestRunnerHooks): - is_finished = False + finished = False results: Queue def update_test(self) -> Union[dict, None]: @@ -31,10 +31,10 @@ def update_test(self) -> Union[dict, None]: return None def is_finished(self) -> bool: - return SDKPythonTestRunnerHooks.is_finished + return SDKPythonTestRunnerHooks.finished def __init__(self) -> None: - SDKPythonTestRunnerHooks.is_finished = False + SDKPythonTestRunnerHooks.finished = False SDKPythonTestRunnerHooks.results = Queue() def start(self, count: int) -> None: diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 1ae9d886..83fd8f3e 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -39,7 +39,7 @@ # Command line params RUNNER_CLASS = "test_harness_client.py" -RUNNER_CLASS_PATH = "/root/python_testing/rpc_client" +RUNNER_CLASS_PATH = "/root/python_testing/rpc_client/" EXECUTABLE = "python3" @@ -159,7 +159,6 @@ async def setup(self) -> None: """Override Setup to log Python Test version.""" test_engine_logger.info(f"Python Test Version: {self.python_test_version}") try: - await super().setup() self.chip_tool = ChipTool() await self.chip_tool.start_container() assert self.chip_tool.is_running() From cee194f44d3a6fc0e25836132b33680a47deb4a0 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 29 Nov 2023 13:29:22 +0000 Subject: [PATCH 31/45] Code changed after code review --- app/chip_tool/chip_tool.py | 21 +++++++++++-------- app/chip_tool/test_suite.py | 1 - app/tests/chip_tool/test_chip_tool.py | 1 - .../python_tests/test_python_test_suite.py | 11 ---------- .../models/python_test_models.py | 4 +++- .../models/python_test_parser.py | 3 ++- .../models/python_testing_hooks_proxy.py | 2 +- .../models/rpc_client/test_harness_client.py | 8 ++++--- .../python_testing/models/test_case.py | 12 +++++------ .../python_testing/models/test_suite.py | 16 -------------- .../yaml_tests/models/yaml_test_models.py | 2 +- 11 files changed, 30 insertions(+), 51 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index d2198eb4..35d5a588 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -108,14 +108,16 @@ "sdk_runner/specifications/chip/" ) # Python Testing Folder -# LOCAL_PYTHON_TESTING_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path( -# "python_testing/scripts/sdk" -# ) -# DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" +LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections" +LOCAL_PYTHON_TESTING_PATH = Path( + LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/sdk_checkout/python_testing/scripts/sdk" +) +DOCKER_PYTHON_TESTING_PATH = "/root/python_testing" # RPC Client Running on SDK Container LOCAL_RPC_PYTHON_TESTING_PATH = Path( - "/home/ubuntu/certification-tool/backend/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py" + LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/support/python_testing/models/rpc_client/" + "test_harness_client.py" ) DOCKER_RPC_PYTHON_TESTING_PATH = "/root/python_testing/test_harness_client.py" @@ -143,7 +145,6 @@ class ChipToolUnknownTestType(Exception): class ChipToolTestType(str, Enum): CHIP_TOOL = "chip-tool" CHIP_APP = "chip-app" - PYTHON_TEST = "python-test" class ChipTool(metaclass=Singleton): @@ -180,6 +181,10 @@ class ChipTool(metaclass=Singleton): "bind": DOCKER_CREDENTIALS_DEVELOPMENT_PATH, "mode": "ro", }, + LOCAL_PYTHON_TESTING_PATH: { + "bind": DOCKER_PYTHON_TESTING_PATH, + "mode": "rw", + }, LOCAL_RPC_PYTHON_TESTING_PATH: { "bind": DOCKER_RPC_PYTHON_TESTING_PATH, "mode": "rw", @@ -412,8 +417,6 @@ async def start_server( async def destroy_device(self) -> None: """Destroy the device container.""" - await self.stop_chip_tool_server() - if self.__chip_tool_container is not None: container_manager.destroy(self.__chip_tool_container) self.__chip_tool_container = None @@ -607,7 +610,7 @@ async def run_test( if test_type == ChipToolTestType.CHIP_TOOL: test_path = f"{YAML_TESTS_PATH}/{test_id}.yaml" - elif test_type == ChipToolTestType.CHIP_APP: + else: test_path = f"{YAML_TESTS_PATH}/{test_id}_Simulated.yaml" parser_config = TestParserConfig(pics_path, self.specifications, test_options) diff --git a/app/chip_tool/test_suite.py b/app/chip_tool/test_suite.py index bed94057..e548427d 100644 --- a/app/chip_tool/test_suite.py +++ b/app/chip_tool/test_suite.py @@ -280,4 +280,3 @@ async def __prompt_user_to_perform_decommission(self) -> None: f"Received unknown prompt option for \ decommissioning step: {prompt_response.response}" ) - diff --git a/app/tests/chip_tool/test_chip_tool.py b/app/tests/chip_tool/test_chip_tool.py index 2c64a843..ae8f522d 100644 --- a/app/tests/chip_tool/test_chip_tool.py +++ b/app/tests/chip_tool/test_chip_tool.py @@ -988,4 +988,3 @@ async def test_pairing_ble_thread_command_params() -> None: # clean up: chip_tool._ChipTool__chip_tool_container = None settings.CHIP_TOOL_TRACE = original_trace_setting_value - diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py index 5a230bab..acdbaccb 100644 --- a/app/tests/python_tests/test_python_test_suite.py +++ b/app/tests/python_tests/test_python_test_suite.py @@ -24,7 +24,6 @@ from app.models.test_suite_execution import TestSuiteExecution from app.test_engine.logger import test_engine_logger from test_collections.sdk_tests.support.python_testing.models.test_suite import ( - ChipToolPythonTestSuite, PythonTestSuite, SuiteType, ) @@ -58,16 +57,6 @@ def test_python_test_suite_python_version() -> None: assert suite_class.python_test_version == python_test_version -def test_automated_suite_subclass() -> None: - """Test that for suite type automated class factory creates a subclass of - ChipToolPythonTestSuite, and that test_type is set to CHIP_TOOL""" - type = SuiteType.AUTOMATED - # Create a subclass of PythonTestSuite - suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( - suite_type=type, name="SomeSuite", python_test_version="some_version" - ) - - @pytest.mark.asyncio async def test_suite_setup_log_python_version() -> None: """Test that test suite python version is logged to test engine logger in setup.""" diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index a8e8bc4d..26c52cf8 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -23,6 +23,8 @@ class PythonTest(MatterTest): + description: str + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.type: MatterTestType.AUTOMATED # type: ignore + self.type = MatterTestType.AUTOMATED diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 3c048492..455cd025 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -82,7 +82,8 @@ def __parse_test_case_from_class( ) from si return PythonTest( - name=tc_desc, + name=tc_name, + description=tc_desc, steps=tc_steps, config=tc_config, PICS=tc_pics, diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index 5dce6255..1f563839 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -42,7 +42,7 @@ def start(self, count: int) -> None: def stop(self, duration: int) -> None: self.results.put({"stop": {"duration": duration}}) - self.is_finished = True + SDKPythonTestRunnerHooks.finished = True def test_start(self, filename: str, name: str, count: int) -> None: self.results.put( diff --git a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py index 415083cf..43c70cbf 100644 --- a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py +++ b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -19,7 +19,7 @@ import sys from multiprocessing.managers import BaseManager -from matter_testing_support import MatterTestConfig, run_tests +import matter_testing_support try: from matter_yamltests.hooks import TestRunnerHooks @@ -33,7 +33,7 @@ class TestRunnerHooks: # Pre-computed param list for each Python Test as defined in Verification Steps. test_params = { - "TC_ACE_1_3": MatterTestConfig( + "TC_ACE_1_3": matter_testing_support.MatterTestConfig( tests=["test_TC_ACE_1_3"], commissioning_method="on-network", discriminators=[3840], @@ -46,6 +46,8 @@ class TestRunnerHooks: def main(): + sys.path.append("/root/python_testing") + if len(sys.argv) != 2: raise Exception("Python test id should be provided as the only parameter.") @@ -64,7 +66,7 @@ def main(): manager.connect() test_runner_hooks = manager.TestRunnerHooks() # shared object proxy - run_tests(TestClassReference, config, test_runner_hooks) + matter_testing_support.run_tests(TestClassReference, config, test_runner_hooks) if __name__ == "__main__": diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 83fd8f3e..ea1ce989 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -39,7 +39,7 @@ # Command line params RUNNER_CLASS = "test_harness_client.py" -RUNNER_CLASS_PATH = "/root/python_testing/rpc_client/" +RUNNER_CLASS_PATH = "/root/python_testing/" EXECUTABLE = "python3" @@ -59,6 +59,7 @@ class PythonTestCase(TestCase): def __init__(self, test_case_execution: TestCaseExecution) -> None: super().__init__(test_case_execution=test_case_execution) self.chip_tool: ChipTool + self.__runned = 0 def start(self, count: int) -> None: pass @@ -177,7 +178,7 @@ def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: @classmethod def __class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: """Common class factory method for all subclasses of PythonTestCase.""" - identifier = cls.__test_identifier(test.name) + identifier = test.name class_name = cls.__class_name(identifier) title = identifier @@ -189,10 +190,10 @@ def __class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: "python_test_version": python_test_version, "chip_tool_test_identifier": class_name, "metadata": { - "public_id": identifier, + "public_id": test.name, "version": "0.0.1", "title": title, - "description": test.name, + "description": test.description, }, }, ) @@ -239,7 +240,7 @@ async def execute(self) -> None: continue self.__handle_update(update) - await sleep(0.1) + await sleep(0.001) finally: pass @@ -257,7 +258,6 @@ def __call_function_from_name(self, obj, func_name, kwargs) -> None: # type: ig async def cleanup(self) -> None: logger.info("Test Cleanup") - self.chip_tool.destroy_device() class PythonChipToolTestCase(PythonTestCase): diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index f2c25b95..9b3753f6 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -47,16 +47,6 @@ def class_factory( cls, suite_type: SuiteType, name: str, python_test_version: str ) -> Type[T]: """Dynamically declares a subclass based on the type of test suite.""" - suite_class = ChipToolPythonTestSuite - - return suite_class.__class_factory( - name=name, python_test_version=python_test_version - ) - - @classmethod - def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: - """Common class factory method for all subclasses of PythonTestSuite.""" - return type( name, (cls,), @@ -71,9 +61,3 @@ def __class_factory(cls, name: str, python_test_version: str) -> Type[T]: }, }, ) - - -class ChipToolPythonTestSuite(PythonTestSuite): - async def setup(self) -> None: - """Due top multi inheritance, we need to call setup on both super classes.""" - await PythonTestSuite.setup(self) diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py index be5208d2..e1d43e37 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py @@ -27,4 +27,4 @@ class YamlTest(YamlModelMixin, MatterTest): def __init__(self, **kwargs: Any) -> None: super().__init__(steps=kwargs["tests"], **kwargs) - self.type: MatterTestType.MANUAL # type: ignore + self.type = MatterTestType.MANUAL From 01d9d57e90b11b7db1690598d6bf077f1b742690 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <116586593+rquidute@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:40:18 -0300 Subject: [PATCH 32/45] Update app/tests/python_tests/test_python_test_suite.py Co-authored-by: Carolina Lopes <116589288+ccruzagralopes@users.noreply.github.com> --- app/tests/python_tests/test_python_test_suite.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py index acdbaccb..8609673a 100644 --- a/app/tests/python_tests/test_python_test_suite.py +++ b/app/tests/python_tests/test_python_test_suite.py @@ -82,9 +82,8 @@ async def test_suite_setup_log_python_version() -> None: @pytest.mark.asyncio async def test_chip_tool_suite_setup() -> None: - """Test that both PythonTestSuite.setup and ChipToolSuite.setup are called when - PythonChipToolsSuite.setup is called. We do this as PythonChipToolsSuite inherits from - both PythonTestSuite and ChipToolSuite.""" + """Test that PythonTestSuite.setup is called when PythonChipToolsSuite.setup is called. + We do this as PythonChipToolsSuite inherits from PythonTestSuite.""" suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( suite_type=SuiteType.AUTOMATED, From 030da70c050474944bbd0fb2b5139676ddf1003a Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 29 Nov 2023 22:36:42 +0000 Subject: [PATCH 33/45] Code changes after review --- app/chip_tool/chip_tool.py | 12 +- .../python_tests/test_python_test_case.py | 22 ---- .../test_sdk_python_collection.py | 2 +- cspell.json | 4 +- .../support/python_testing/__init__.py | 2 +- .../models/python_test_parser.py | 7 +- .../models/python_testing_hooks_proxy.py | 8 +- .../models/rpc_client/test_harness_client.py | 6 +- .../python_testing/models/test_case.py | 117 ++---------------- ...thon_test_tests.py => sdk_python_tests.py} | 0 10 files changed, 30 insertions(+), 150 deletions(-) rename test_collections/sdk_tests/support/python_testing/{sdk_python_test_tests.py => sdk_python_tests.py} (100%) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 35d5a588..5190163b 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -373,6 +373,12 @@ async def start_container(self) -> None: Creates the chip-tool container without any server running (ChipTool or ChipApp). """ + if self.is_running(): + self.logger.info( + "chip-tool container already running, no need to start a new container" + ) + return + # Ensure there's no existing container running using the same name. self.__destroy_existing_container() # Async return when the container is running @@ -401,12 +407,6 @@ async def start_server( Returns only when the container is created and all chip-tool services start. """ - if self.is_running(): - self.logger.info( - "chip-tool container already running, no need to start a new container" - ) - return - await self.start_container() web_socket_config = WebSocketRunnerConfig() diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index 4f62c7bc..0c268dd6 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -33,9 +33,6 @@ from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( PythonTest, ) -from test_collections.sdk_tests.support.python_testing.models.test_case import ( - PythonChipToolTestCase, -) def python_test_instance( @@ -126,16 +123,6 @@ def test_python_test_case_class_default_test_parameters() -> None: assert case_class.default_test_parameters() == expected_default_test_parameters -def test_automated_test_case_class_factory_subclass_mapping() -> None: - """Test Automated tests are created as a subclass of - PythonChipToolTestCase.""" - test = python_test_instance(type=MatterTestType.AUTOMATED) - case_class: Type[PythonTestCase] = PythonTestCase.class_factory( - test=test, python_test_version="version" - ) - assert issubclass(case_class, PythonChipToolTestCase) - - def test_class_factory_test_public_id() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1' in python test name. And set it as public_id in metadata""" @@ -176,15 +163,6 @@ def test_class_factory_test_class_name() -> None: assert case_class.__name__ == data["class_name"] -def test_test_type_for_automated_tests() -> None: - """Test that automated tests are set to use chip-tool""" - test = python_test_instance(type=MatterTestType.AUTOMATED) - case_class: Type[PythonTestCase] = PythonTestCase.class_factory( - test=test, python_test_version="version" - ) - assert issubclass(case_class, PythonChipToolTestCase) - - @pytest.mark.asyncio async def test_python_version_logging() -> None: """Test that all Python tests will log Python test version to test_engine_logger. diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index 3a5c6a68..b969fc37 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -28,7 +28,7 @@ PythonCaseDeclaration, PythonCollectionDeclaration, ) -from test_collections.sdk_tests.support.python_testing.sdk_python_test_tests import ( +from test_collections.sdk_tests.support.python_testing.sdk_python_tests import ( sdk_python_test_collection, ) diff --git a/cspell.json b/cspell.json index 28ac7526..5963d04b 100644 --- a/cspell.json +++ b/cspell.json @@ -89,7 +89,9 @@ "isnot", "DBAPI", "repr", - "zinfo" + "zinfo", + "MCORE", + "elts" ], "allowCompoundWords": true, "ignorePaths": [ diff --git a/test_collections/sdk_tests/support/python_testing/__init__.py b/test_collections/sdk_tests/support/python_testing/__init__.py index 31921266..138004de 100644 --- a/test_collections/sdk_tests/support/python_testing/__init__.py +++ b/test_collections/sdk_tests/support/python_testing/__init__.py @@ -15,7 +15,7 @@ # from app.test_engine.models.test_declarations import TestCollectionDeclaration -from .sdk_python_test_tests import sdk_python_test_collection +from .sdk_python_tests import sdk_python_test_collection # Test engine will auto load TestCollectionDeclarations declared inside the package # initializer diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 455cd025..ef8408dd 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -38,11 +38,6 @@ def parse_python_test(path: Path) -> PythonTest: This will also annotate parsed python test with it's path and test type. """ - - return __parse_test_case_from_file(path) - - -def __parse_test_case_from_file(path: Path) -> PythonTest: with open(path, "r") as python_file: parsed_python_file = ast.parse(python_file.read()) classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] @@ -51,7 +46,7 @@ def __parse_test_case_from_file(path: Path) -> PythonTest: try: class_ = next(c for c in classes if tc_name in c.name) except StopIteration as si: # Raised when `next` doesn't find a matching method - raise PythonParserException(f"{path} must have a class name {tc_name}") from si + raise PythonParserException(f"{path} must have a class named {tc_name}") from si return __parse_test_case_from_class(class_=class_, path=path, tc_name=tc_name) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index 1f563839..a418d5cd 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -23,6 +23,10 @@ class SDKPythonTestRunnerHooks(TestRunnerHooks): finished = False results: Queue + def __init__(self) -> None: + SDKPythonTestRunnerHooks.finished = False + SDKPythonTestRunnerHooks.results = Queue() + def update_test(self) -> Union[dict, None]: try: result = self.results.get(block=False) @@ -33,10 +37,6 @@ def update_test(self) -> Union[dict, None]: def is_finished(self) -> bool: return SDKPythonTestRunnerHooks.finished - def __init__(self) -> None: - SDKPythonTestRunnerHooks.finished = False - SDKPythonTestRunnerHooks.results = Queue() - def start(self, count: int) -> None: self.results.put({"start": {"count": count}}) diff --git a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py index 43c70cbf..f9b75b42 100644 --- a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py +++ b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# type: ignore +# flake8: noqa import importlib import sys @@ -45,7 +47,7 @@ class TestRunnerHooks: } -def main(): +def main() -> None: sys.path.append("/root/python_testing") if len(sys.argv) != 2: @@ -64,7 +66,7 @@ def main(): BaseManager.register(TestRunnerHooks.__name__) manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") manager.connect() - test_runner_hooks = manager.TestRunnerHooks() # shared object proxy + test_runner_hooks = manager.TestRunnerHooks() # shared object proxy # type: ignore matter_testing_support.run_tests(TestClassReference, config, test_runner_hooks) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index ea1ce989..8cb8d802 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -13,22 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os -import re from asyncio import sleep from multiprocessing.managers import BaseManager from typing import Any, Type, TypeVar -from matter_chip_tool_adapter.decoder import MatterLog - from app.chip_tool.chip_tool import ChipTool from app.models import TestCaseExecution -from app.test_engine.logger import ( - CHIP_LOG_FORMAT, - CHIPTOOL_LEVEL, - logger, - test_engine_logger, -) +from app.test_engine.logger import logger, test_engine_logger from app.test_engine.models import TestCase, TestStep from .python_test_models import PythonTest @@ -60,6 +51,7 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: super().__init__(test_case_execution=test_case_execution) self.chip_tool: ChipTool self.__runned = 0 + self.test_stop_called = False def start(self, count: int) -> None: pass @@ -69,9 +61,7 @@ def stop(self, duration: int) -> None: self.current_test_step.mark_as_completed() def test_start(self, filename: str, name: str, count: int) -> None: - pass - # Dont know if it is necessary for python testing (came from chip_tool) - # self.next_step() + self.next_step() def test_stop(self, exception: Exception, duration: int) -> None: self.test_stop_called = True @@ -87,75 +77,23 @@ def step_start(self, name: str) -> None: pass def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: - self.__handle_logs(logs) + # TODO Handle Logs properly self.next_step() def step_failure( self, logger: Any, logs: str, duration: int, request: Any, received: Any ) -> None: - self.__handle_logs(logs) - self.__report_failures(logger, request, received) + # TODO Handle Logs properly self.next_step() def step_unknown(self) -> None: self.__runned += 1 - def __handle_logs(self, logs: Any) -> None: - for log_entry in logs or []: - if not isinstance(log_entry, MatterLog): - continue - - test_engine_logger.log( - CHIPTOOL_LEVEL, - CHIP_LOG_FORMAT.format(log_entry.module, log_entry.message), - ) - - def __report_failures(self, logger: Any, request: TestStep, received: Any) -> None: - """ - The logger from runner contains all logs entries for the test step, this method - seeks for the error entries. - """ - if not logger: - # It is expected the runner to return a PostProcessResponseResult, - # but in case of returning a different type - self.current_test_step.append_failure( - "Test Step Failure: \n Expected: '<Empty>' \n Received: '<Empty>'" - ) - return - - # Iterate through the entries seeking for the errors entries - for log_entry in logger.entries or []: - if log_entry.is_error(): - # Check if the step error came from exception or not, since the message - # in exception object has more details - # TODO: There is an issue raised in SDK runner in order to improve the - # message from log_entry: - # https://github.com/project-chip/connectedhomeip/issues/28101 - if log_entry.exception: - self.current_test_step.append_failure(log_entry.exception.message) - else: - self.current_test_step.append_failure(log_entry.message) - @classmethod def pics(cls) -> set[str]: """Test Case level PICS. Read directly from parsed Python Test.""" return cls.python_test.PICS - @classmethod - def default_test_parameters(cls) -> dict[str, Any]: - """Python Testing config dict, sometimes have a nested dict with type and - default value. - Only defaultValue is used in this case. - """ - parameters = {} - for param_name, value in cls.python_test.config.items(): - if isinstance(value, dict): - if "defaultValue" in value: - parameters[param_name] = value["defaultValue"] - else: - parameters[param_name] = value - return parameters - async def setup(self) -> None: """Override Setup to log Python Test version.""" test_engine_logger.info(f"Python Test Version: {self.python_test_version}") @@ -168,55 +106,23 @@ async def setup(self) -> None: @classmethod def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: - """Dynamically declares a subclass based on the type of Python Test test.""" - case_class: Type[PythonTestCase] = PythonChipToolTestCase - - return case_class.__class_factory( - test=test, python_test_version=python_test_version - ) - - @classmethod - def __class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: - """Common class factory method for all subclasses of PythonTestCase.""" - identifier = test.name - class_name = cls.__class_name(identifier) - title = identifier - + """class factory method for PythonTestCase.""" return type( - class_name, + test.name, (cls,), { "python_test": test, "python_test_version": python_test_version, - "chip_tool_test_identifier": class_name, + "chip_tool_test_identifier": test.name, "metadata": { "public_id": test.name, "version": "0.0.1", - "title": title, + "title": test.name, "description": test.description, }, }, ) - @staticmethod - def __test_identifier(name: str) -> str: - """Find TC-XX-1.1 in Python Test title. - Note some have [TC-XX-1.1] and others TC-XX-1.1 - """ - title_pattern = re.compile(r"(?P<title>TC-[^\s\]]*)") - if match := re.search(title_pattern, name): - return match["title"] - else: - return name - - @staticmethod - def __class_name(identifier: str) -> str: - """Replace all non-alphanumeric characters with _ to make valid class name.""" - return re.sub("[^0-9a-zA-Z]+", "_", identifier) - - def run_command(self, cmd: str) -> None: - os.system(cmd) - async def execute(self) -> None: try: logger.info("Running Python Test: " + self.metadata["title"]) @@ -259,11 +165,8 @@ def __call_function_from_name(self, obj, func_name, kwargs) -> None: # type: ig async def cleanup(self) -> None: logger.info("Test Cleanup") - -class PythonChipToolTestCase(PythonTestCase): - """Automated Python test cases""" - def create_test_steps(self) -> None: + self.test_steps = [TestStep("Start Python test")] for step in self.python_test.steps: python_test_step = TestStep(step.label) self.test_steps.append(python_test_step) diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_tests.py similarity index 100% rename from test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py rename to test_collections/sdk_tests/support/python_testing/sdk_python_tests.py From de0c0c4fc92fbf841deb9e3f3139a561c91b3d2d Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 29 Nov 2023 22:42:09 +0000 Subject: [PATCH 34/45] Remove whitespace from a blank line --- app/chip_tool/chip_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 5190163b..7d5fc5ee 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -378,7 +378,7 @@ async def start_container(self) -> None: "chip-tool container already running, no need to start a new container" ) return - + # Ensure there's no existing container running using the same name. self.__destroy_existing_container() # Async return when the container is running From 0a769fb87f67225e2c949553c6e8e891f9b6f9d0 Mon Sep 17 00:00:00 2001 From: Carolina Lopes <ccruzagralopes@apple.com> Date: Thu, 30 Nov 2023 21:27:03 +0000 Subject: [PATCH 35/45] Add support for PICS in python tests --- app/chip_tool/chip_tool.py | 32 ++++++++------ app/chip_tool/test_suite.py | 2 +- app/tests/chip_tool/test_chip_tool.py | 4 +- .../models/rpc_client/test_harness_client.py | 29 ++----------- .../python_testing/models/test_case.py | 43 ++++++++++--------- .../python_testing/models/test_suite.py | 26 +++++++++-- 6 files changed, 72 insertions(+), 64 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index 7d5fc5ee..c3539c33 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -224,6 +224,10 @@ def __init__( specifications_paths, self.pseudo_clusters ) + @property + def pics_file_created(self) -> bool: + return self.__pics_file_created + @property def node_id(self) -> int: """Node id is used to reference DUT during testing. @@ -683,32 +687,36 @@ def __trace_file_params(self, topic: str) -> str: path = Path(DOCKER_LOGS_PATH) / filename return f'--trace_file "{path}" --trace_decode 1' - def set_pics(self, pics: PICS) -> None: - """Sends command to chip tool to create pics file inside the container. + def set_pics(self, pics: PICS, in_container: bool) -> None: + """Sends command to create pics file. Args: pics (PICS): PICS that contains all the pics codes + in_container (bool): Whether the file should be created in the container or + not. If false, the file is created on the host. Raises: - ChipToolNotRunning: Raises exception if chip tool is not running. PICSError: If creating PICS file inside the container fails. - """ # List of default PICS which needs to set specifically in TH are added here. # These PICS are applicable for CI / Chip tool testing purposes only. # These PICS are unknown / not visible to external users. pics_codes = self.__pics_file_content(pics) + "\n".join(DEFAULT_PICS) - cmd = f"{SHELL_PATH} {SHELL_OPTION} " - cmd = cmd + f"\"{ECHO_COMMAND} '{pics_codes}\n' > {PICS_FILE_PATH}\"" - self.logger.info(f"Sending command: {cmd}") - result = subprocess.run(cmd, shell=True) - # When streaming logs, the exit code is not directly available. - # By storing the execution id, the exit code can be fetched from docker later. - self.__last_exec_id = str(result.returncode) + prefix = f"{SHELL_PATH} {SHELL_OPTION}" + cmd = f"\"{ECHO_COMMAND} '{pics_codes}' > {PICS_FILE_PATH}\"" + + if in_container: + exec_result = self.send_command(cmd, prefix=prefix) + success = exec_result.exit_code == 0 + else: + full_cmd = f"{prefix} {cmd}" + self.logger.info(f"Sending command: {full_cmd}") + result = subprocess.run(full_cmd, shell=True) + success = result.returncode == 0 - if result.returncode != 0: + if not success: raise PICSError("Creating PICS file failed") self.__pics_file_created = True diff --git a/app/chip_tool/test_suite.py b/app/chip_tool/test_suite.py index e548427d..57f0c1e5 100644 --- a/app/chip_tool/test_suite.py +++ b/app/chip_tool/test_suite.py @@ -60,7 +60,7 @@ async def setup(self) -> None: if len(self.pics.clusters) > 0: logger.info("Create PICS file for DUT") - self.chip_tool.set_pics(pics=self.pics) + self.chip_tool.set_pics(pics=self.pics, in_container=False) else: # Disable sending "-PICS" option when running chip-tool self.chip_tool.reset_pics_state() diff --git a/app/tests/chip_tool/test_chip_tool.py b/app/tests/chip_tool/test_chip_tool.py index ae8f522d..58f7555a 100644 --- a/app/tests/chip_tool/test_chip_tool.py +++ b/app/tests/chip_tool/test_chip_tool.py @@ -454,7 +454,7 @@ async def test_set_pics() -> None: ) as mock_run: await chip_tool.start_server(test_type) - chip_tool.set_pics(pics) + chip_tool.set_pics(pics, in_container=False) mock_run.assert_called_once_with(expected_command, shell=True) assert chip_tool._ChipTool__pics_file_created is True @@ -473,7 +473,7 @@ def test_set_pics_with_error() -> None: target="app.chip_tool.chip_tool.subprocess.run", return_value=CompletedProcess("", 1), ), pytest.raises(PICSError): - chip_tool.set_pics(pics) + chip_tool.set_pics(pics, in_container=False) assert chip_tool._ChipTool__pics_file_created is False # clean up: diff --git a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py index f9b75b42..16e927eb 100644 --- a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py +++ b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -23,39 +23,18 @@ import matter_testing_support -try: - from matter_yamltests.hooks import TestRunnerHooks -except: - class TestRunnerHooks: - pass - - -MATTER_DEVELOPMENT_PAA_ROOT_CERTS = "/paa-root-certs" - -# Pre-computed param list for each Python Test as defined in Verification Steps. -test_params = { - "TC_ACE_1_3": matter_testing_support.MatterTestConfig( - tests=["test_TC_ACE_1_3"], - commissioning_method="on-network", - discriminators=[3840], - setup_passcodes=[20202021], - dut_node_ids=[0x12344321], - paa_trust_store_path=MATTER_DEVELOPMENT_PAA_ROOT_CERTS, - storage_path="/root/admin_storage.json", - ) -} +class TestRunnerHooks: + pass def main() -> None: sys.path.append("/root/python_testing") - if len(sys.argv) != 2: - raise Exception("Python test id should be provided as the only parameter.") - test_name = sys.argv[1] - config = test_params.get(test_name) + config_options = sys.argv[2:] + config = matter_testing_support.parse_matter_test_args(config_options) if config is None: raise ValueError(f"Not a valid test id: {test_name}") diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 8cb8d802..c31e410b 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -17,9 +17,9 @@ from multiprocessing.managers import BaseManager from typing import Any, Type, TypeVar -from app.chip_tool.chip_tool import ChipTool +from app.chip_tool.chip_tool import PICS_FILE_PATH, ChipTool from app.models import TestCaseExecution -from app.test_engine.logger import logger, test_engine_logger +from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestCase, TestStep from .python_test_models import PythonTest @@ -49,7 +49,7 @@ class PythonTestCase(TestCase): def __init__(self, test_case_execution: TestCaseExecution) -> None: super().__init__(test_case_execution=test_case_execution) - self.chip_tool: ChipTool + self.chip_tool: ChipTool = ChipTool(logger) self.__runned = 0 self.test_stop_called = False @@ -68,9 +68,7 @@ def test_stop(self, exception: Exception, duration: int) -> None: self.current_test_step.mark_as_completed() def step_skipped(self, name: str, expression: str) -> None: - self.current_test_step.mark_as_not_applicable( - f"Test step skipped: {name}. {expression} == False" - ) + self.current_test_step.mark_as_not_applicable("Test step skipped") self.next_step() def step_start(self, name: str) -> None: @@ -94,16 +92,6 @@ def pics(cls) -> set[str]: """Test Case level PICS. Read directly from parsed Python Test.""" return cls.python_test.PICS - async def setup(self) -> None: - """Override Setup to log Python Test version.""" - test_engine_logger.info(f"Python Test Version: {self.python_test_version}") - try: - self.chip_tool = ChipTool() - await self.chip_tool.start_container() - assert self.chip_tool.is_running() - except NotImplementedError: - pass - @classmethod def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: """class factory method for PythonTestCase.""" @@ -123,17 +111,35 @@ def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: }, ) + async def setup(self) -> None: + logger.info("Test Setup") + + async def cleanup(self) -> None: + logger.info("Test Cleanup") + async def execute(self) -> None: try: logger.info("Running Python Test: " + self.metadata["title"]) + BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") manager.start() test_runner_hooks = manager.TestRunnerHooks() # type: ignore + runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS + command = ( + f"{runner_class} {self.metadata['title']}" + " --commissioning-method on-network --discriminator 3840 --passcode" + " 20202021 --storage-path /root/admin_storage.json" + " --paa-trust-store-path /paa-root-certs" + ) + + if self.chip_tool.pics_file_created: + command += f" --PICS {PICS_FILE_PATH}" + # TODO Ignoring stream from docker execution self.chip_tool.send_command( - f"{runner_class} {self.metadata['title']}", + command, prefix=EXECUTABLE, is_stream=True, is_socket=False, @@ -162,9 +168,6 @@ def __call_function_from_name(self, obj, func_name, kwargs) -> None: # type: ig raise TypeError(f"{func_name} is not callable") func(**kwargs) - async def cleanup(self) -> None: - logger.info("Test Cleanup") - def create_test_steps(self) -> None: self.test_steps = [TestStep("Start Python test")] for step in self.python_test.steps: diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 9b3753f6..18e3e44c 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -16,6 +16,7 @@ from enum import Enum from typing import Type, TypeVar +from app.chip_tool import ChipTool from app.test_engine.logger import test_engine_logger as logger from app.test_engine.models import TestSuite @@ -37,10 +38,7 @@ class PythonTestSuite(TestSuite): python_test_version: str suite_name: str - - async def setup(self) -> None: - """Override Setup to log Python Test version.""" - logger.info(f"Python Test Version: {self.python_test_version}") + chip_tool: ChipTool = ChipTool(logger) @classmethod def class_factory( @@ -61,3 +59,23 @@ def class_factory( }, }, ) + + async def setup(self) -> None: + """Override Setup to log Python Test version and set PICS.""" + logger.info("Suite Setup") + logger.info(f"Python Test Version: {self.python_test_version}") + + logger.info("Starting SDK container") + await self.chip_tool.start_container() + + if len(self.pics.clusters) > 0: + logger.info("Create PICS file for DUT") + self.chip_tool.set_pics(pics=self.pics, in_container=True) + else: + self.chip_tool.reset_pics_state() + + async def cleanup(self) -> None: + logger.info("Suite Cleanup") + + logger.info("Stopping SDK container") + await self.chip_tool.destroy_device() From c7ad322565fb4fe91db5d63708c70b0291d6f626 Mon Sep 17 00:00:00 2001 From: Carolina Lopes <ccruzagralopes@apple.com> Date: Thu, 30 Nov 2023 22:22:46 +0000 Subject: [PATCH 36/45] Update some unit tests --- app/tests/chip_tool/test_chip_tool.py | 5 +- .../python_tests/test_python_test_suite.py | 77 +++++++++++++++++-- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/app/tests/chip_tool/test_chip_tool.py b/app/tests/chip_tool/test_chip_tool.py index 58f7555a..bb65e96c 100644 --- a/app/tests/chip_tool/test_chip_tool.py +++ b/app/tests/chip_tool/test_chip_tool.py @@ -111,7 +111,6 @@ async def test_start_container_using_paa_certs() -> None: @pytest.mark.asyncio async def test_not_start_container_when_running() -> None: chip_tool = ChipTool() - test_type = ChipToolTestType.CHIP_TOOL with mock.patch.object( target=chip_tool, attribute="is_running", return_value=True @@ -120,7 +119,7 @@ async def test_not_start_container_when_running() -> None: ) as mock_create_container, mock.patch.object( target=chip_tool, attribute="start_chip_server" ) as mock_start_chip_server: - await chip_tool.start_server(test_type) + await chip_tool.start_container() mock_create_container.assert_not_called() mock_start_chip_server.assert_not_called() @@ -432,7 +431,7 @@ async def test_set_pics() -> None: "PICS_USER_PROMPT=1" ) expected_command = ( - f"{SHELL_PATH} {SHELL_OPTION} \"echo '{expected_pics_data}\n' " + f"{SHELL_PATH} {SHELL_OPTION} \"echo '{expected_pics_data}' " f'> {PICS_FILE_PATH}"' ) diff --git a/app/tests/python_tests/test_python_test_suite.py b/app/tests/python_tests/test_python_test_suite.py index 8609673a..199775ef 100644 --- a/app/tests/python_tests/test_python_test_suite.py +++ b/app/tests/python_tests/test_python_test_suite.py @@ -20,9 +20,11 @@ import pytest -from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.chip_tool import ChipTool from app.models.test_suite_execution import TestSuiteExecution +from app.schemas import PICS from app.test_engine.logger import test_engine_logger +from app.tests.utils.test_pics_data import create_random_pics from test_collections.sdk_tests.support.python_testing.models.test_suite import ( PythonTestSuite, SuiteType, @@ -60,6 +62,8 @@ def test_python_test_suite_python_version() -> None: @pytest.mark.asyncio async def test_suite_setup_log_python_version() -> None: """Test that test suite python version is logged to test engine logger in setup.""" + chip_tool: ChipTool = ChipTool() + for type in list(SuiteType): python_test_version = "best_version" # Create a subclass of PythonTestSuite @@ -69,17 +73,80 @@ async def test_suite_setup_log_python_version() -> None: suite_instance = suite_class(TestSuiteExecution()) - # We're patching ChipToolSuite.setup to avoid starting chip-tool with mock.patch.object( target=test_engine_logger, attribute="info" - ) as logger_info, mock.patch( - "app.chip_tool.test_suite.ChipToolSuite.setup" - ) as _: + ) as logger_info, mock.patch.object( + target=chip_tool, attribute="start_container" + ), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.test_suite" + ".PythonTestSuite.pics", + new_callable=PICS, + ): await suite_instance.setup() logger_info.assert_called() logger_info.assert_any_call(f"Python Test Version: {python_test_version}") +@pytest.mark.asyncio +async def test_suite_setup_without_pics() -> None: + chip_tool: ChipTool = ChipTool() + + for type in list(SuiteType): + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version=python_test_version + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch( + "app.chip_tool.test_suite.ChipToolSuite.setup" + ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.test_suite" + ".PythonTestSuite.pics", + new_callable=PICS, + ), mock.patch.object( + target=chip_tool, attribute="set_pics" + ) as mock_set_pics, mock.patch.object( + target=chip_tool, attribute="reset_pics_state" + ) as mock_reset_pics_state: + await suite_instance.setup() + + mock_set_pics.assert_not_called() + mock_reset_pics_state.assert_called_once() + + +@pytest.mark.asyncio +async def test_suite_setup_with_pics() -> None: + chip_tool: ChipTool = ChipTool() + + for type in list(SuiteType): + python_test_version = "best_version" + # Create a subclass of PythonTestSuite + suite_class: Type[PythonTestSuite] = PythonTestSuite.class_factory( + suite_type=type, name="SomeSuite", python_test_version=python_test_version + ) + + suite_instance = suite_class(TestSuiteExecution()) + + with mock.patch( + "app.chip_tool.test_suite.ChipToolSuite.setup" + ), mock.patch.object(target=chip_tool, attribute="start_container"), mock.patch( + target="test_collections.sdk_tests.support.python_testing.models.test_suite" + ".PythonTestSuite.pics", + new_callable=create_random_pics, + ), mock.patch.object( + target=chip_tool, attribute="set_pics" + ) as mock_set_pics, mock.patch.object( + target=chip_tool, attribute="reset_pics_state" + ) as mock_reset_pics_state: + await suite_instance.setup() + + mock_set_pics.assert_called_once() + mock_reset_pics_state.assert_not_called() + + @pytest.mark.asyncio async def test_chip_tool_suite_setup() -> None: """Test that PythonTestSuite.setup is called when PythonChipToolsSuite.setup is called. From 5904c9d2e2ab7e24eba434d53255ad911891564b Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Fri, 1 Dec 2023 17:39:41 +0000 Subject: [PATCH 37/45] Get python test parameters from project config --- app/tests/python_tests/test_python_parser.py | 2 +- .../python_tests/test_python_test_case.py | 12 +++- .../test_python_test_declarations.py | 4 +- .../python_testing/models/test_case.py | 67 +++++++++++++++++-- .../yaml_tests/models/yaml_test_models.py | 3 +- 5 files changed, 74 insertions(+), 14 deletions(-) diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py index c4eb37c2..5f128891 100644 --- a/app/tests/python_tests/test_python_parser.py +++ b/app/tests/python_tests/test_python_parser.py @@ -73,7 +73,7 @@ def test_python_file_parser_throws_pythonparserexception() -> None: try: parse_python_test(file_path) except PythonParserException as e: - assert "/test/file.py must have a class name file" == str(e) + assert "/test/file.py must have a class named file" == str(e) def test_python_file_parser() -> None: diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index 0c268dd6..881db114 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -37,6 +37,7 @@ def python_test_instance( name: str = "Test Python", + description: str = "Test Python Description", PICS: set[str] = {"PICS.A", "PICS.B"}, config: dict[str, Any] = { "param1": "value1", @@ -53,19 +54,22 @@ def python_test_instance( steps=steps, type=type, path=path, + description=description, ) def test_python_test_name() -> None: """Test that test name is set as description in metadata.""" name = "Another Test Name" - test = python_test_instance(name=name) + description = "Another Test Name Description" + test = python_test_instance(name=name, description=description) # Create a subclass of PythonTest case_class: Type[PythonTestCase] = PythonTestCase.class_factory( test=test, python_test_version="version" ) - assert case_class.metadata["description"] == name + assert case_class.metadata["title"] == name + assert case_class.metadata["description"] == description def test_python_test_python_version() -> None: @@ -224,7 +228,9 @@ def test_multiple_steps_for_python_tests() -> None: @pytest.mark.asyncio async def test_setup_super_error_handling() -> None: # ignore requirement to create_tests on init - with mock.patch("app.test_engine.models.test_case.TestCase.create_test_steps") as _: + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.create_test_steps" + ) as _: test = PythonTestCase(TestCaseExecution()) test.python_test_version = "some version" # Assert this doesn't raise an exception diff --git a/app/tests/python_tests/test_python_test_declarations.py b/app/tests/python_tests/test_python_test_declarations.py index 5c797a8b..57d660c4 100644 --- a/app/tests/python_tests/test_python_test_declarations.py +++ b/app/tests/python_tests/test_python_test_declarations.py @@ -47,7 +47,9 @@ def test_python_suite_declaration() -> None: def test_python_case_declaration() -> None: - test = PythonTest(name="TestTest", config={}, steps=[]) + test = PythonTest( + name="TestTest", description="TestTest description", config={}, steps=[] + ) version = "SomeVersionStr" with mock.patch( "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.class_factory" diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index c31e410b..32ccdf13 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -117,6 +117,59 @@ async def setup(self) -> None: async def cleanup(self) -> None: logger.info("Test Cleanup") + def __generate_command_arguments(self) -> list: + # All valid arguments for python test + valid_args = [ + "ble_interface_id", + "commissioning_method", + "controller_node_id", + "discriminator", + "endpoint", + "logs_path", + "PICS", + "paa_trust_store_path", + "timeout", + "trace_to", + "int_arg", + "float_arg", + "string_arg", + "json_arg", + "hex_arg", + "bool_arg", + "storage_path", + "passcode", + "dut-node-id", + ] + + dut_config = self.project.config.dut_config + test_parameters = self.project.config.test_parameters + + pairing_mode = ( + "on-network" + if dut_config.pairing_mode == "onnetwork" + else dut_config.pairing_mode + ) + + arguments = [] + # Retrieve arguments from dut_config + arguments.append(f"--discriminator {dut_config.discriminator}") + arguments.append(f"--passcode {dut_config.setup_code}") + arguments.append(f"--commissioning-method {pairing_mode}") + + # Retrieve arguments from test_parameters + + if test_parameters: + for name, value in test_parameters.items(): + if name in valid_args: + if str(value) != "": + arguments.append(f"--{name.replace('_','-')} {str(value)}") + else: + arguments.append(f"--{name.replace('_','-')} " "") + else: + logger.warning(f"Argument {name} is not valid") + + return arguments + async def execute(self) -> None: try: logger.info("Running Python Test: " + self.metadata["title"]) @@ -127,15 +180,15 @@ async def execute(self) -> None: test_runner_hooks = manager.TestRunnerHooks() # type: ignore runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS - command = ( - f"{runner_class} {self.metadata['title']}" - " --commissioning-method on-network --discriminator 3840 --passcode" - " 20202021 --storage-path /root/admin_storage.json" - " --paa-trust-store-path /paa-root-certs" - ) + command = [f"{runner_class} {self.metadata['title']}"] + + # Generate the command argument by getting the test_parameters from + # project configuration + command_arguments = self.__generate_command_arguments() + command.extend(command_arguments) if self.chip_tool.pics_file_created: - command += f" --PICS {PICS_FILE_PATH}" + command.append(f" --PICS {PICS_FILE_PATH}") # TODO Ignoring stream from docker execution self.chip_tool.send_command( diff --git a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py index e1d43e37..65918eee 100644 --- a/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py +++ b/test_collections/sdk_tests/support/yaml_tests/models/yaml_test_models.py @@ -17,7 +17,7 @@ from pydantic_yaml import YamlModelMixin -from ...models.matter_test_models import MatterTest, MatterTestType +from ...models.matter_test_models import MatterTest ### # This file declares YAML models that are used to parse the YAML Test Cases. @@ -27,4 +27,3 @@ class YamlTest(YamlModelMixin, MatterTest): def __init__(self, **kwargs: Any) -> None: super().__init__(steps=kwargs["tests"], **kwargs) - self.type = MatterTestType.MANUAL From 3198ecdc85a28dfcff3b7c1253b18cd81c2d1559 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Fri, 1 Dec 2023 18:58:18 +0000 Subject: [PATCH 38/45] Fix Unit tests --- .../python_tests/test_python_test_case.py | 23 +---- .../python_testing/models/test_case.py | 97 +++++++------------ 2 files changed, 34 insertions(+), 86 deletions(-) diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index 881db114..ffde45c6 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -106,27 +106,6 @@ def test_python_test_case_class_pics() -> None: assert case_class.pics() == test_PICS -def test_python_test_case_class_default_test_parameters() -> None: - """Test that the default_test_parameters of the python test is available in the class - method default_test_parameters on TestCase. - - Also parameters with type in Python test should be flattened and type dropped.""" - - test_input_config = { - "param1": "value1", - "param2": {"type": "config_type", "defaultValue": "value2"}, - } - - test = python_test_instance(config=test_input_config) - expected_default_test_parameters = {"param1": "value1", "param2": "value2"} - - # Create a subclass of PythonTest - case_class: Type[PythonTestCase] = PythonTestCase.class_factory( - test=test, python_test_version="version" - ) - assert case_class.default_test_parameters() == expected_default_test_parameters - - def test_class_factory_test_public_id() -> None: """Test that class factory correctly finds identifier 'TC-XX-1.1' in python test name. And set it as public_id in metadata""" @@ -189,7 +168,7 @@ async def test_python_version_logging() -> None: except TestError: pass logger_info.assert_called() - logger_info.assert_any_call(f"Python Test Version: {test_python_version}") + logger_info.assert_any_call("Test Setup") def test_normal_steps_for_python_tests() -> None: diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 32ccdf13..5ddcfd6a 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import re from asyncio import sleep from multiprocessing.managers import BaseManager from typing import Any, Type, TypeVar @@ -95,81 +96,49 @@ def pics(cls) -> set[str]: @classmethod def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: """class factory method for PythonTestCase.""" + + identifier = cls.__test_identifier(test.name) + class_name = cls.__class_name(identifier) + title = identifier + return type( - test.name, + class_name, (cls,), { "python_test": test, "python_test_version": python_test_version, - "chip_tool_test_identifier": test.name, + "chip_tool_test_identifier": class_name, "metadata": { - "public_id": test.name, + "public_id": identifier, "version": "0.0.1", - "title": test.name, + "title": title, "description": test.description, }, }, ) + @staticmethod + def __test_identifier(name: str) -> str: + """Find TC-XX-1.1 in YAML title. + Note some have [TC-XX-1.1] and others TC-XX-1.1 + """ + title_pattern = re.compile(r"(?P<title>TC-[^\s\]]*)") + if match := re.search(title_pattern, name): + return match["title"] + else: + return name + + @staticmethod + def __class_name(identifier: str) -> str: + """Replace all non-alphanumeric characters with _ to make valid class name.""" + return re.sub("[^0-9a-zA-Z]+", "_", identifier) + async def setup(self) -> None: logger.info("Test Setup") async def cleanup(self) -> None: logger.info("Test Cleanup") - def __generate_command_arguments(self) -> list: - # All valid arguments for python test - valid_args = [ - "ble_interface_id", - "commissioning_method", - "controller_node_id", - "discriminator", - "endpoint", - "logs_path", - "PICS", - "paa_trust_store_path", - "timeout", - "trace_to", - "int_arg", - "float_arg", - "string_arg", - "json_arg", - "hex_arg", - "bool_arg", - "storage_path", - "passcode", - "dut-node-id", - ] - - dut_config = self.project.config.dut_config - test_parameters = self.project.config.test_parameters - - pairing_mode = ( - "on-network" - if dut_config.pairing_mode == "onnetwork" - else dut_config.pairing_mode - ) - - arguments = [] - # Retrieve arguments from dut_config - arguments.append(f"--discriminator {dut_config.discriminator}") - arguments.append(f"--passcode {dut_config.setup_code}") - arguments.append(f"--commissioning-method {pairing_mode}") - - # Retrieve arguments from test_parameters - - if test_parameters: - for name, value in test_parameters.items(): - if name in valid_args: - if str(value) != "": - arguments.append(f"--{name.replace('_','-')} {str(value)}") - else: - arguments.append(f"--{name.replace('_','-')} " "") - else: - logger.warning(f"Argument {name} is not valid") - - return arguments - async def execute(self) -> None: try: logger.info("Running Python Test: " + self.metadata["title"]) @@ -180,15 +149,15 @@ async def execute(self) -> None: test_runner_hooks = manager.TestRunnerHooks() # type: ignore runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS - command = [f"{runner_class} {self.metadata['title']}"] - - # Generate the command argument by getting the test_parameters from - # project configuration - command_arguments = self.__generate_command_arguments() - command.extend(command_arguments) + command = ( + f"{runner_class} {self.metadata['title']}" + " --commissioning-method on-network --discriminator 3840 --passcode" + " 20202021 --storage-path /root/admin_storage.json" + " --paa-trust-store-path /paa-root-certs" + ) if self.chip_tool.pics_file_created: - command.append(f" --PICS {PICS_FILE_PATH}") + command += f" --PICS {PICS_FILE_PATH}" # TODO Ignoring stream from docker execution self.chip_tool.send_command( From f35d043163fed5e967ce4573681998d6f47e33d2 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Mon, 4 Dec 2023 18:26:08 +0000 Subject: [PATCH 39/45] Updated unit tests --- app/tests/python_tests/test_sdk_python_collection.py | 10 +--------- app/tests/yaml_tests/test_sdk_yaml_collection.py | 5 +---- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py index b969fc37..7e157cae 100644 --- a/app/tests/python_tests/test_sdk_python_collection.py +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -49,15 +49,7 @@ def test_sdk_python_test_collection( ) -> None: assert python_test_collection.name == "SDK Python Tests" assert len(python_test_collection.test_suites.keys()) == 1 - - # test version number - test_sdk_python_version_path = ( - "/app/backend/app/tests/python_tests/test_python_script/.version" - ) - with open(test_sdk_python_version_path, "r") as version_file: - assert ( - python_test_collection.python_test_version == version_file.read().rstrip() - ) + assert python_test_collection.python_test_version == "unit-test-python-version" def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> None: diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index 71a88864..0c02fe2e 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -48,10 +48,7 @@ def test_sdk_yaml_collection( assert yaml_collection.name == "SDK YAML Tests" assert len(yaml_collection.test_suites.keys()) == 3 - # test version number - test_sdk_yaml_version_path = "/app/backend/app/tests/yaml_tests/test_yamls/.version" - with open(test_sdk_yaml_version_path, "r") as version_file: - assert yaml_collection.yaml_version == version_file.read().rstrip() + assert yaml_collection.yaml_version == "unit-test-yaml-version" def test_manual_suite(yaml_collection: YamlCollectionDeclaration) -> None: From 9bb424fccf5a04dd25a2cb2c963872d614e36977 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Mon, 4 Dec 2023 18:33:41 +0000 Subject: [PATCH 40/45] Removed unused test files --- app/tests/python_tests/test_python_script/.version | 1 - app/tests/yaml_tests/test_sdk_yaml_collection.py | 1 - app/tests/yaml_tests/test_yamls/.version | 1 - 3 files changed, 3 deletions(-) delete mode 100644 app/tests/python_tests/test_python_script/.version delete mode 100644 app/tests/yaml_tests/test_yamls/.version diff --git a/app/tests/python_tests/test_python_script/.version b/app/tests/python_tests/test_python_script/.version deleted file mode 100644 index 3e269350..00000000 --- a/app/tests/python_tests/test_python_script/.version +++ /dev/null @@ -1 +0,0 @@ -unit-test-python-version \ No newline at end of file diff --git a/app/tests/yaml_tests/test_sdk_yaml_collection.py b/app/tests/yaml_tests/test_sdk_yaml_collection.py index 0c02fe2e..3f67d063 100644 --- a/app/tests/yaml_tests/test_sdk_yaml_collection.py +++ b/app/tests/yaml_tests/test_sdk_yaml_collection.py @@ -47,7 +47,6 @@ def test_sdk_yaml_collection( ) -> None: assert yaml_collection.name == "SDK YAML Tests" assert len(yaml_collection.test_suites.keys()) == 3 - assert yaml_collection.yaml_version == "unit-test-yaml-version" diff --git a/app/tests/yaml_tests/test_yamls/.version b/app/tests/yaml_tests/test_yamls/.version deleted file mode 100644 index 0fc82cc3..00000000 --- a/app/tests/yaml_tests/test_yamls/.version +++ /dev/null @@ -1 +0,0 @@ -unit-test-yaml-version \ No newline at end of file From e5dfd63525cec19b0b34fa13d356a4eb75c2bfbd Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Tue, 5 Dec 2023 18:18:14 +0000 Subject: [PATCH 41/45] Code updated due to code review --- app/chip_tool/chip_tool.py | 5 ++- .../python_tests/test_python_test_case.py | 15 +------ .../models/python_test_parser.py | 11 +++++ .../models/rpc_client/test_harness_client.py | 2 +- .../python_testing/models/test_case.py | 40 +++++++++---------- 5 files changed, 37 insertions(+), 36 deletions(-) diff --git a/app/chip_tool/chip_tool.py b/app/chip_tool/chip_tool.py index c3539c33..2e3970bd 100644 --- a/app/chip_tool/chip_tool.py +++ b/app/chip_tool/chip_tool.py @@ -692,8 +692,9 @@ def set_pics(self, pics: PICS, in_container: bool) -> None: Args: pics (PICS): PICS that contains all the pics codes - in_container (bool): Whether the file should be created in the container or - not. If false, the file is created on the host. + in_container (bool): Whether the file should be created in the SDK container + or not. YAML tests run directly in the backend and + python tests run in the SDK container. Raises: PICSError: If creating PICS file inside the container fails. diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py index ffde45c6..78a303d8 100644 --- a/app/tests/python_tests/test_python_test_case.py +++ b/app/tests/python_tests/test_python_test_case.py @@ -36,7 +36,7 @@ def python_test_instance( - name: str = "Test Python", + name: str = "TC-Test-Python", description: str = "Test Python Description", PICS: set[str] = {"PICS.A", "PICS.B"}, config: dict[str, Any] = { @@ -111,12 +111,6 @@ def test_class_factory_test_public_id() -> None: And set it as public_id in metadata""" test_data = [ {"name": "TC-AB-1.2", "public_id": "TC-AB-1.2"}, - {"name": "[TC-CD-3.4]", "public_id": "TC-CD-3.4"}, - {"name": "Test Name before [TC-EF-5.6]", "public_id": "TC-EF-5.6"}, - {"name": "[TC-GH-7.8] Test Name after", "public_id": "TC-GH-7.8"}, - {"name": "Before and [TC-IJ-9.0] after", "public_id": "TC-IJ-9.0"}, - {"name": "Before and TC-KL-10.11 after", "public_id": "TC-KL-10.11"}, - {"name": "TC-MORE-NAME-13.110", "public_id": "TC-MORE-NAME-13.110"}, ] for data in test_data: test = python_test_instance(name=data["name"]) @@ -131,12 +125,7 @@ def test_class_factory_test_class_name() -> None: a safe class name, eg TC_XX_1_1""" test_data = [ {"name": "TC-AB-1.2", "class_name": "TC_AB_1_2"}, - {"name": "[TC-CD-3.4]", "class_name": "TC_CD_3_4"}, - {"name": "Test Name before [TC-EF-5.6]", "class_name": "TC_EF_5_6"}, - {"name": "[TC-GH-7.8] Test Name after", "class_name": "TC_GH_7_8"}, - {"name": "Before and [TC-IJ-9.0] after", "class_name": "TC_IJ_9_0"}, - {"name": "Before and TC-KL-10.11 after", "class_name": "TC_KL_10_11"}, - {"name": "TC-MORE-NAME-13.110", "class_name": "TC_MORE_NAME_13_110"}, + {"name": "TC-CD-3.4", "class_name": "TC_CD_3_4"}, ] for data in test_data: test = python_test_instance(name=data["name"]) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index ef8408dd..622953b2 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -37,6 +37,17 @@ def parse_python_test(path: Path) -> PythonTest: """Parse a single Python test file into PythonTest model. This will also annotate parsed python test with it's path and test type. + + It's expected that the python script files has a class with the same file name, + and also the following methods: + * desc_[class_name] - This method should return a str with the test description + * pics_[class_name] - This method should return a list with the PICS required + for the test case + * steps_[class_name] - This method should return a list with the step description + * test_[class_name] - This method contains the test logic + + Example: file TC_ACE_1_3.py must have the methods desc_TC_ACE_1_3, pics_TC_ACE_1_3, + steps_TC_ACE_1_3 and test_TC_ACE_1_3. """ with open(path, "r") as python_file: parsed_python_file = ast.parse(python_file.read()) diff --git a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py index 16e927eb..57eb0757 100644 --- a/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py +++ b/test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py @@ -29,8 +29,8 @@ class TestRunnerHooks: def main() -> None: + # Load python_testing as a module. This folder is where all python script is located sys.path.append("/root/python_testing") - test_name = sys.argv[1] config_options = sys.argv[2:] diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 5ddcfd6a..0883e4b2 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -96,10 +96,8 @@ def pics(cls) -> set[str]: @classmethod def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: """class factory method for PythonTestCase.""" - - identifier = cls.__test_identifier(test.name) - class_name = cls.__class_name(identifier) - title = identifier + title = cls.__title(test.name) + class_name = cls.__class_name(test.name) return type( class_name, @@ -107,9 +105,8 @@ def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: { "python_test": test, "python_test_version": python_test_version, - "chip_tool_test_identifier": class_name, "metadata": { - "public_id": identifier, + "public_id": test.name, "version": "0.0.1", "title": title, "description": test.description, @@ -117,22 +114,24 @@ def class_factory(cls, test: PythonTest, python_test_version: str) -> Type[T]: }, ) - @staticmethod - def __test_identifier(name: str) -> str: - """Find TC-XX-1.1 in YAML title. - Note some have [TC-XX-1.1] and others TC-XX-1.1 - """ - title_pattern = re.compile(r"(?P<title>TC-[^\s\]]*)") - if match := re.search(title_pattern, name): - return match["title"] - else: - return name - @staticmethod def __class_name(identifier: str) -> str: """Replace all non-alphanumeric characters with _ to make valid class name.""" return re.sub("[^0-9a-zA-Z]+", "_", identifier) + @staticmethod + def __title(identifier: str) -> str: + """Retrieve the test title in format TC_ABC_1_2""" + title: str = "" + elements = identifier.split("_") + + if len(elements) > 2: + title = "-".join(elements[0:2]) + "-" + ".".join(elements[2:]) + else: + title = identifier.replace("_", "-") + + return title + async def setup(self) -> None: logger.info("Test Setup") @@ -141,7 +140,7 @@ async def cleanup(self) -> None: async def execute(self) -> None: try: - logger.info("Running Python Test: " + self.metadata["title"]) + logger.info("Running Python Test: " + self.python_test.name) BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") @@ -150,7 +149,7 @@ async def execute(self) -> None: runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS command = ( - f"{runner_class} {self.metadata['title']}" + f"{runner_class} {self.python_test.name}" " --commissioning-method on-network --discriminator 3840 --passcode" " 20202021 --storage-path /root/admin_storage.json" " --paa-trust-store-path /paa-root-certs" @@ -171,10 +170,11 @@ async def execute(self) -> None: not test_runner_hooks.is_finished() ): if not update: + await sleep(0.0001) continue self.__handle_update(update) - await sleep(0.001) + finally: pass From 1e1ae772a32941b4550dea91302ab914b34d30b2 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 6 Dec 2023 14:44:39 +0000 Subject: [PATCH 42/45] Make PICS optional for python tests --- .../python_testing/models/python_test_parser.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 622953b2..da2f2c15 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -79,14 +79,18 @@ def __parse_test_case_from_class( steps_method = next(m for m in methods if steps_method_name in m.name) tc_steps = __retrieve_steps(steps_method) - - pics_method = next(m for m in methods if pics_method_name in m.name) - tc_pics = __retrieve_pics(pics_method) except StopIteration as si: # Raised when `next` doesn't find a matching method raise PythonParserException( f"{path} did not contain valid definition for {tc_name}" ) from si + # PICS method is optional + try: + pics_method = next(m for m in methods if pics_method_name in m.name) + tc_pics = __retrieve_pics(pics_method) + except StopIteration: # Raised when `next` doesn't find a matching method + tc_pics = [] + return PythonTest( name=tc_name, description=tc_desc, From 890cf2e1cfba4c2bbbd3e39daf351f858b8d4b08 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 6 Dec 2023 16:54:40 +0000 Subject: [PATCH 43/45] Added SDKPythonTestResult class in order to handle the python test responses --- .../models/python_testing_hooks_proxy.py | 100 +++++++++++++----- .../python_testing/models/test_case.py | 31 +++--- 2 files changed, 93 insertions(+), 38 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index a418d5cd..f8c0a6ec 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -13,10 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from enum import Enum from queue import Empty, Queue -from typing import Any, Union +from typing import Any, Optional, Union from matter_yamltests.hooks import TestRunnerHooks +from pydantic import BaseModel + + +class SDKPythonTestResultEnum(str, Enum): + START = "start" + STOP = "stop" + TEST_START = "test_start" + TEST_STOP = "test_stop" + STEP_SKIPPED = "step_skipped" + STEP_START = "step_start" + STEP_SUCCESS = "step_success" + STEP_FAILURE = "step_failure" + STEP_UNKNOWN = "step_unknown" + STEP_MANUAL = "step_manual" + + +class SDKPythonTestResult(BaseModel): + type: SDKPythonTestResultEnum + count: Optional[int] + duration: Optional[str] + name: Optional[str] + filename: Optional[str] + exception: Any + expression: Optional[str] + logger: Any + logs: Any + request: Any + received: Any class SDKPythonTestRunnerHooks(TestRunnerHooks): @@ -38,58 +67,77 @@ def is_finished(self) -> bool: return SDKPythonTestRunnerHooks.finished def start(self, count: int) -> None: - self.results.put({"start": {"count": count}}) + self.results.put( + SDKPythonTestResult(type=SDKPythonTestResultEnum.START, count=count) + ) def stop(self, duration: int) -> None: - self.results.put({"stop": {"duration": duration}}) + self.results.put( + SDKPythonTestResult(type=SDKPythonTestResultEnum.STOP, duration=duration) + ) SDKPythonTestRunnerHooks.finished = True def test_start(self, filename: str, name: str, count: int) -> None: self.results.put( - {"test_start": {"filename": filename, "name": name, "count": count}} + SDKPythonTestResult( + type=SDKPythonTestResultEnum.TEST_START, + filename=filename, + name=name, + count=count, + ) ) def test_stop(self, exception: Exception, duration: int) -> None: - self.results.put({"test_stop": {"exception": exception, "duration": duration}}) + self.results.put( + SDKPythonTestResult( + type=SDKPythonTestResultEnum.TEST_STOP, + exception=exception, + duration=duration, + ) + ) def step_skipped(self, name: str, expression: str) -> None: - self.results.put({"step_skipped": {"name": name, "expression": expression}}) + self.results.put( + SDKPythonTestResult( + type=SDKPythonTestResultEnum.STEP_SKIPPED, expression=expression + ) + ) def step_start(self, name: str) -> None: - self.results.put({"step_start": {"name": name}}) + self.results.put( + SDKPythonTestResult(type=SDKPythonTestResultEnum.STEP_START, name=name) + ) def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: self.results.put( - { - "step_success": { - "logger": logger, - "logs": logs, - "duration": duration, - "request": request, - } - } + SDKPythonTestResult( + type=SDKPythonTestResultEnum.STEP_SUCCESS, + logger=logger, + logs=logs, + duration=duration, + request=request, + ) ) def step_failure( self, logger: Any, logs: Any, duration: int, request: Any, received: Any ) -> None: self.results.put( - { - "step_failure": { - "logger": logger, - "logs": logs, - "duration": duration, - "request": request, - "received": received, - } - } + SDKPythonTestResult( + type=SDKPythonTestResultEnum.STEP_FAILURE, + logger=logger, + logs=logs, + duration=duration, + request=request, + received=received, + ) ) def step_unknown(self) -> None: - self.results.put({"step_unknown": {}}) + self.results.put(SDKPythonTestResult(type=SDKPythonTestResultEnum.STEP_UNKNOWN)) def step_manual(self) -> None: - self.results.put({"step_manual": {}}) + self.results.put(SDKPythonTestResult(type=SDKPythonTestResultEnum.STEP_MANUAL)) def step_start_list(self) -> None: pass diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 0883e4b2..9f7490bd 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -24,7 +24,7 @@ from app.test_engine.models import TestCase, TestStep from .python_test_models import PythonTest -from .python_testing_hooks_proxy import SDKPythonTestRunnerHooks +from .python_testing_hooks_proxy import SDKPythonTestResult, SDKPythonTestRunnerHooks # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") @@ -54,33 +54,41 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: self.__runned = 0 self.test_stop_called = False - def start(self, count: int) -> None: + def start(self, count: int, **kwargs: Any) -> None: pass - def stop(self, duration: int) -> None: + def stop(self, duration: int, **kwargs: Any) -> None: if not self.test_stop_called: self.current_test_step.mark_as_completed() - def test_start(self, filename: str, name: str, count: int) -> None: + def test_start(self, filename: str, name: str, count: int, **kwargs: Any) -> None: self.next_step() - def test_stop(self, exception: Exception, duration: int) -> None: + def test_stop(self, exception: Exception, duration: int, **kwargs: Any) -> None: self.test_stop_called = True self.current_test_step.mark_as_completed() - def step_skipped(self, name: str, expression: str) -> None: + def step_skipped(self, name: str, expression: str, **kwargs: Any) -> None: self.current_test_step.mark_as_not_applicable("Test step skipped") self.next_step() - def step_start(self, name: str) -> None: + def step_start(self, name: str, **kwargs: Any) -> None: pass - def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: + def step_success( + self, logger: Any, logs: str, duration: int, request: Any, **kwargs: Any + ) -> None: # TODO Handle Logs properly self.next_step() def step_failure( - self, logger: Any, logs: str, duration: int, request: Any, received: Any + self, + logger: Any, + logs: str, + duration: int, + request: Any, + received: Any, + **kwargs: Any, ) -> None: # TODO Handle Logs properly self.next_step() @@ -178,9 +186,8 @@ async def execute(self) -> None: finally: pass - def __handle_update(self, update: dict) -> None: - for func_name, kwargs in update.items(): - self.__call_function_from_name(self, func_name, kwargs) + def __handle_update(self, update: SDKPythonTestResult) -> None: + self.__call_function_from_name(self, update.type.value, update.__dict__) def __call_function_from_name(self, obj, func_name, kwargs) -> None: # type: ignore func = getattr(obj, func_name, None) From 8a116d1d45b79afecb480e06697df066351f17d6 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 6 Dec 2023 18:04:58 +0000 Subject: [PATCH 44/45] Create SDKPythonTestResultBase and make subclass for each result --- .../models/python_testing_hooks_proxy.py | 90 +++++++++++++++---- .../python_testing/models/test_case.py | 40 ++++----- 2 files changed, 93 insertions(+), 37 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index f8c0a6ec..2936c77f 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -15,7 +15,7 @@ # from enum import Enum from queue import Empty, Queue -from typing import Any, Optional, Union +from typing import Any, Literal, Optional, Union from matter_yamltests.hooks import TestRunnerHooks from pydantic import BaseModel @@ -34,20 +34,72 @@ class SDKPythonTestResultEnum(str, Enum): STEP_MANUAL = "step_manual" -class SDKPythonTestResult(BaseModel): +class SDKPythonTestResultBase(BaseModel): type: SDKPythonTestResultEnum + + def params_dict(self) -> dict: + return self.dict(exclude={"type"}) + + +class SDKPythonTestResultStart(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.START] + count: int + + +class SDKPythonTestResultStop(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STOP] + duration: int + + +class SDKPythonTestResultTestStart(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.TEST_START] + filename: Optional[str] + name: Optional[str] count: Optional[int] + + +class SDKPythonTestResultTestStop(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.TEST_STOP] duration: Optional[str] - name: Optional[str] - filename: Optional[str] exception: Any + + +class SDKPythonTestResultStepSkipped(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STEP_SKIPPED] + name: Optional[str] expression: Optional[str] + + +class SDKPythonTestResultStepStart(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STEP_START] + name: Optional[str] + + +class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STEP_SUCCESS] logger: Any logs: Any + duration: int + request: Any + + +class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STEP_FAILURE] + logger: Any + logs: Any + duration: int request: Any received: Any +class SDKPythonTestResultStepUnknown(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STEP_UNKNOWN] + + +class SDKPythonTestResultStepManual(SDKPythonTestResultBase): + type: Literal[SDKPythonTestResultEnum.STEP_MANUAL] + + class SDKPythonTestRunnerHooks(TestRunnerHooks): finished = False results: Queue @@ -68,18 +120,20 @@ def is_finished(self) -> bool: def start(self, count: int) -> None: self.results.put( - SDKPythonTestResult(type=SDKPythonTestResultEnum.START, count=count) + SDKPythonTestResultStart(type=SDKPythonTestResultEnum.START, count=count) ) def stop(self, duration: int) -> None: self.results.put( - SDKPythonTestResult(type=SDKPythonTestResultEnum.STOP, duration=duration) + SDKPythonTestResultStop( + type=SDKPythonTestResultEnum.STOP, duration=duration + ) ) SDKPythonTestRunnerHooks.finished = True def test_start(self, filename: str, name: str, count: int) -> None: self.results.put( - SDKPythonTestResult( + SDKPythonTestResultTestStart( type=SDKPythonTestResultEnum.TEST_START, filename=filename, name=name, @@ -89,28 +143,30 @@ def test_start(self, filename: str, name: str, count: int) -> None: def test_stop(self, exception: Exception, duration: int) -> None: self.results.put( - SDKPythonTestResult( + SDKPythonTestResultTestStop( type=SDKPythonTestResultEnum.TEST_STOP, exception=exception, duration=duration, ) ) - def step_skipped(self, name: str, expression: str) -> None: + def SDKPythonTestResultStepSkipped(self, name: str, expression: str) -> None: self.results.put( - SDKPythonTestResult( + SDKPythonTestResultStepSkipped( type=SDKPythonTestResultEnum.STEP_SKIPPED, expression=expression ) ) def step_start(self, name: str) -> None: self.results.put( - SDKPythonTestResult(type=SDKPythonTestResultEnum.STEP_START, name=name) + SDKPythonTestResultStepStart( + type=SDKPythonTestResultEnum.STEP_START, name=name + ) ) def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: self.results.put( - SDKPythonTestResult( + SDKPythonTestResultStepSuccess( type=SDKPythonTestResultEnum.STEP_SUCCESS, logger=logger, logs=logs, @@ -123,7 +179,7 @@ def step_failure( self, logger: Any, logs: Any, duration: int, request: Any, received: Any ) -> None: self.results.put( - SDKPythonTestResult( + SDKPythonTestResultStepFailure( type=SDKPythonTestResultEnum.STEP_FAILURE, logger=logger, logs=logs, @@ -134,10 +190,14 @@ def step_failure( ) def step_unknown(self) -> None: - self.results.put(SDKPythonTestResult(type=SDKPythonTestResultEnum.STEP_UNKNOWN)) + self.results.put( + SDKPythonTestResultStepUnknown(type=SDKPythonTestResultEnum.STEP_UNKNOWN) + ) def step_manual(self) -> None: - self.results.put(SDKPythonTestResult(type=SDKPythonTestResultEnum.STEP_MANUAL)) + self.results.put( + SDKPythonTestResultStepManual(type=SDKPythonTestResultEnum.STEP_MANUAL) + ) def step_start_list(self) -> None: pass diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index 9f7490bd..0dff2cd1 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -24,7 +24,10 @@ from app.test_engine.models import TestCase, TestStep from .python_test_models import PythonTest -from .python_testing_hooks_proxy import SDKPythonTestResult, SDKPythonTestRunnerHooks +from .python_testing_hooks_proxy import ( + SDKPythonTestResultBase, + SDKPythonTestRunnerHooks, +) # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") @@ -54,43 +57,36 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None: self.__runned = 0 self.test_stop_called = False - def start(self, count: int, **kwargs: Any) -> None: + def start(self, count: int) -> None: pass - def stop(self, duration: int, **kwargs: Any) -> None: + def stop(self, duration: int) -> None: if not self.test_stop_called: self.current_test_step.mark_as_completed() - def test_start(self, filename: str, name: str, count: int, **kwargs: Any) -> None: + def test_start(self, filename: str, name: str, count: int) -> None: self.next_step() - def test_stop(self, exception: Exception, duration: int, **kwargs: Any) -> None: + def test_stop(self, exception: Exception, duration: int) -> None: self.test_stop_called = True self.current_test_step.mark_as_completed() - def step_skipped(self, name: str, expression: str, **kwargs: Any) -> None: + def step_skipped(self, name: str, expression: str) -> None: self.current_test_step.mark_as_not_applicable("Test step skipped") self.next_step() - def step_start(self, name: str, **kwargs: Any) -> None: + def step_start(self, name: str) -> None: pass - def step_success( - self, logger: Any, logs: str, duration: int, request: Any, **kwargs: Any - ) -> None: + def step_success(self, logger: Any, logs: str, duration: int, request: Any) -> None: # TODO Handle Logs properly self.next_step() def step_failure( - self, - logger: Any, - logs: str, - duration: int, - request: Any, - received: Any, - **kwargs: Any, + self, logger: Any, logs: str, duration: int, request: Any, received: Any ) -> None: # TODO Handle Logs properly + self.mark_step_failure("Python test step failure") self.next_step() def step_unknown(self) -> None: @@ -186,13 +182,13 @@ async def execute(self) -> None: finally: pass - def __handle_update(self, update: SDKPythonTestResult) -> None: - self.__call_function_from_name(self, update.type.value, update.__dict__) + def __handle_update(self, update: SDKPythonTestResultBase) -> None: + self.__call_function_from_name(update.type.value, update.params_dict()) - def __call_function_from_name(self, obj, func_name, kwargs) -> None: # type: ignore - func = getattr(obj, func_name, None) + def __call_function_from_name(self, func_name: str, kwargs: Any) -> None: + func = getattr(self, func_name, None) if not func: - raise AttributeError(f"{func_name} is not a method of {obj}") + raise AttributeError(f"{func_name} is not a method of {self}") if not callable(func): raise TypeError(f"{func_name} is not callable") func(**kwargs) From eceb15b8e3272cb4d24c14608b9a49cc4702a72e Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho <rquidute@apple.com> Date: Wed, 6 Dec 2023 19:27:28 +0000 Subject: [PATCH 45/45] Make default attribute type set in SDKPythonTestResultBase subclasses --- .../models/python_testing_hooks_proxy.py | 56 ++++++------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py index 2936c77f..78bd878e 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_testing_hooks_proxy.py @@ -15,7 +15,7 @@ # from enum import Enum from queue import Empty, Queue -from typing import Any, Literal, Optional, Union +from typing import Any, Optional, Union from matter_yamltests.hooks import TestRunnerHooks from pydantic import BaseModel @@ -42,41 +42,41 @@ def params_dict(self) -> dict: class SDKPythonTestResultStart(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.START] + type = SDKPythonTestResultEnum.START count: int class SDKPythonTestResultStop(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STOP] + type = SDKPythonTestResultEnum.STOP duration: int class SDKPythonTestResultTestStart(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.TEST_START] + type = SDKPythonTestResultEnum.TEST_START filename: Optional[str] name: Optional[str] count: Optional[int] class SDKPythonTestResultTestStop(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.TEST_STOP] + type = SDKPythonTestResultEnum.TEST_STOP duration: Optional[str] exception: Any class SDKPythonTestResultStepSkipped(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STEP_SKIPPED] + type = SDKPythonTestResultEnum.STEP_SKIPPED name: Optional[str] expression: Optional[str] class SDKPythonTestResultStepStart(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STEP_START] + type = SDKPythonTestResultEnum.STEP_START name: Optional[str] class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STEP_SUCCESS] + type = SDKPythonTestResultEnum.STEP_SUCCESS logger: Any logs: Any duration: int @@ -84,7 +84,7 @@ class SDKPythonTestResultStepSuccess(SDKPythonTestResultBase): class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STEP_FAILURE] + type = SDKPythonTestResultEnum.STEP_FAILURE logger: Any logs: Any duration: int @@ -93,11 +93,11 @@ class SDKPythonTestResultStepFailure(SDKPythonTestResultBase): class SDKPythonTestResultStepUnknown(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STEP_UNKNOWN] + type = SDKPythonTestResultEnum.STEP_UNKNOWN class SDKPythonTestResultStepManual(SDKPythonTestResultBase): - type: Literal[SDKPythonTestResultEnum.STEP_MANUAL] + type = SDKPythonTestResultEnum.STEP_MANUAL class SDKPythonTestRunnerHooks(TestRunnerHooks): @@ -119,22 +119,15 @@ def is_finished(self) -> bool: return SDKPythonTestRunnerHooks.finished def start(self, count: int) -> None: - self.results.put( - SDKPythonTestResultStart(type=SDKPythonTestResultEnum.START, count=count) - ) + self.results.put(SDKPythonTestResultStart(count=count)) def stop(self, duration: int) -> None: - self.results.put( - SDKPythonTestResultStop( - type=SDKPythonTestResultEnum.STOP, duration=duration - ) - ) + self.results.put(SDKPythonTestResultStop(duration=duration)) SDKPythonTestRunnerHooks.finished = True def test_start(self, filename: str, name: str, count: int) -> None: self.results.put( SDKPythonTestResultTestStart( - type=SDKPythonTestResultEnum.TEST_START, filename=filename, name=name, count=count, @@ -144,30 +137,20 @@ def test_start(self, filename: str, name: str, count: int) -> None: def test_stop(self, exception: Exception, duration: int) -> None: self.results.put( SDKPythonTestResultTestStop( - type=SDKPythonTestResultEnum.TEST_STOP, exception=exception, duration=duration, ) ) def SDKPythonTestResultStepSkipped(self, name: str, expression: str) -> None: - self.results.put( - SDKPythonTestResultStepSkipped( - type=SDKPythonTestResultEnum.STEP_SKIPPED, expression=expression - ) - ) + self.results.put(SDKPythonTestResultStepSkipped(expression=expression)) def step_start(self, name: str) -> None: - self.results.put( - SDKPythonTestResultStepStart( - type=SDKPythonTestResultEnum.STEP_START, name=name - ) - ) + self.results.put(SDKPythonTestResultStepStart(name=name)) def step_success(self, logger: Any, logs: Any, duration: int, request: Any) -> None: self.results.put( SDKPythonTestResultStepSuccess( - type=SDKPythonTestResultEnum.STEP_SUCCESS, logger=logger, logs=logs, duration=duration, @@ -180,7 +163,6 @@ def step_failure( ) -> None: self.results.put( SDKPythonTestResultStepFailure( - type=SDKPythonTestResultEnum.STEP_FAILURE, logger=logger, logs=logs, duration=duration, @@ -190,14 +172,10 @@ def step_failure( ) def step_unknown(self) -> None: - self.results.put( - SDKPythonTestResultStepUnknown(type=SDKPythonTestResultEnum.STEP_UNKNOWN) - ) + self.results.put(SDKPythonTestResultStepUnknown()) def step_manual(self) -> None: - self.results.put( - SDKPythonTestResultStepManual(type=SDKPythonTestResultEnum.STEP_MANUAL) - ) + self.results.put(SDKPythonTestResultStepManual()) def step_start_list(self) -> None: pass