From fe24421ef3d335512a24cecfbe85d36a8bbf6613 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Wed, 8 Nov 2023 16:56:38 +0000 Subject: [PATCH] Added Unit tests --- app/tests/python_tests/test_python_folder.py | 59 ++++ app/tests/python_tests/test_python_parser.py | 87 ++++++ .../python_tests/test_python_script/.version | 1 + .../UnitTest_TC_ACL_1_1_Automated.py | 51 ++++ .../python_tests/test_python_test_case.py | 259 ++++++++++++++++++ .../test_python_test_declarations.py | 55 ++++ .../test_sdk_python_collection.py | 74 +++++ .../models/python_test_models.py | 2 +- .../models/python_test_parser.py | 15 +- .../python_testing/models/test_case.py | 25 +- .../python_testing/models/test_suite.py | 5 +- .../python_testing/sdk_python_test_tests.py | 20 +- 12 files changed, 609 insertions(+), 44 deletions(-) create mode 100644 app/tests/python_tests/test_python_folder.py create mode 100644 app/tests/python_tests/test_python_parser.py create mode 100644 app/tests/python_tests/test_python_script/.version create mode 100644 app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py create mode 100644 app/tests/python_tests/test_python_test_case.py create mode 100644 app/tests/python_tests/test_python_test_declarations.py create mode 100644 app/tests/python_tests/test_sdk_python_collection.py diff --git a/app/tests/python_tests/test_python_folder.py b/app/tests/python_tests/test_python_folder.py new file mode 100644 index 00000000..69e2bade --- /dev/null +++ b/app/tests/python_tests/test_python_folder.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.python_test_folder import ( + PythonTestFolder, +) + +test_python_path = Path("/test/python") + + +def test_python_folder_version() -> None: + version_file_content = "python_test_version" + + # We mock open to read version_file_content and Path exists to ignore that we're + # testing with a fake path + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_folder.open", + new=mock.mock_open(read_data=version_file_content), + ), mock.patch.object(target=Path, attribute="exists", return_value=True) as _: + python_test_folder = PythonTestFolder(test_python_path) + + assert python_test_folder.version == version_file_content + + +def test_python_folder_version_missing() -> None: + expected_version = "Unknown" + with mock.patch.object(target=Path, attribute="exists", return_value=False) as _: + python_folder = PythonTestFolder(test_python_path) + assert python_folder.version == expected_version + + +def test_python_folder_filename_pattern() -> None: + """Test PythonTestFolder will search for files with filename pattern.""" + with mock.patch.object(target=Path, attribute="glob") as path_glob: + # Default file_name_patter: * + python_folder = PythonTestFolder(test_python_path) + _ = python_folder.python_file_paths() + path_glob.assert_called_once_with("*.py") + + path_glob.reset_mock() + pattern = "TC_*" + python_test_folder = PythonTestFolder(test_python_path, filename_pattern=pattern) + _ = python_test_folder.python_file_paths() + path_glob.assert_called_once_with(f"{pattern}.py") diff --git a/app/tests/python_tests/test_python_parser.py b/app/tests/python_tests/test_python_parser.py new file mode 100644 index 00000000..a85c8e36 --- /dev/null +++ b/app/tests/python_tests/test_python_parser.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.python_test_parser import ( + PythonParserException, + parse_python_test, +) + +sample_invalid_python_file_content = """ +class TC_Sample(MatterBaseTest): + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + @async_test_body + async def test_steps_TC_Sample(self): + print("Test execution") + +""" + +sample_python_file_content = """ +class TC_Sample(MatterBaseTest): + + def desc_TC_Sample(self) -> str: + return "Sample TC Description" + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + @async_test_body + async def test_steps_TC_Sample(self): + print("Test execution") + +""" + + +def test_python_file_parser_throws_pythonparserexception() -> None: + file_path = Path("/test/file.py") + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_parser.open", + new=mock.mock_open(read_data=sample_invalid_python_file_content), + ): + try: + parse_python_test(file_path) + except PythonParserException as e: + assert "Test Case file does not have methods desc_file or steps_file" == str(e) + + +def test_python_file_parser() -> None: + file_path = Path("/test/file.py") + + # We mock builtin `open` method to read sample python file content, + # to avoid having to load a real file. + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.python_test_parser.open", + new=mock.mock_open(read_data=sample_python_file_content), + ) as file_open: + test = parse_python_test(file_path) + + file_open.assert_called_once_with(file_path, "r") + assert test.path == file_path diff --git a/app/tests/python_tests/test_python_script/.version b/app/tests/python_tests/test_python_script/.version new file mode 100644 index 00000000..3e269350 --- /dev/null +++ b/app/tests/python_tests/test_python_script/.version @@ -0,0 +1 @@ +unit-test-python-version \ No newline at end of file diff --git a/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py new file mode 100644 index 00000000..261a6bfe --- /dev/null +++ b/app/tests/python_tests/test_python_script/UnitTest_TC_ACL_1_1_Automated.py @@ -0,0 +1,51 @@ + + # + # Copyright (c) 2023 Project CHIP Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# type: ignore +# Ignore mypy type check for this file + +""" +This is just a sample test case that should come from SDK. +It should not compile or run. +""" +class TC_Sample(MatterBaseTest): + + def desc_TC_Sample(self) -> str: + return "Sample TC Description" + + def steps_TC_Sample(self) -> list[TestStep]: + steps = [ + TestStep(1, "Commissioning, already done", is_commissioning=True), + TestStep(2, "Second step"), + TestStep(3, "Third step"), + ] + return steps + + @async_test_body + async def test_steps_TC_Sample(self): + print("Test execution") \ No newline at end of file diff --git a/app/tests/python_tests/test_python_test_case.py b/app/tests/python_tests/test_python_test_case.py new file mode 100644 index 00000000..addedd91 --- /dev/null +++ b/app/tests/python_tests/test_python_test_case.py @@ -0,0 +1,259 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path +from typing import Any, Optional, Type +from unittest import mock + +import pytest + +from app.chip_tool.chip_tool import ChipToolTestType +from app.chip_tool.test_case import TestError +from app.models.test_case_execution import TestCaseExecution +from app.test_engine.logger import test_engine_logger +from test_collections.sdk_tests.support.python_testing.models import PythonTestCase +from test_collections.sdk_tests.support.python_testing.models.test_case import ( + PythonChipToolTestCase, +) +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, + PythonTestStep, + PythonTestType, +) + + +def python_test_instance( + name: str = "Test Python", + PICS: set[str] = {"PICS.A", "PICS.B"}, + config: dict[str, Any] = { + "param1": "value1", + "param2": {"type": "config_type", "defaultValue": "value2"}, + }, + steps: list[PythonTestStep] = [], + type: PythonTestType = PythonTestType.AUTOMATED, + path: Optional[Path] = None, +) -> PythonTest: + return PythonTest( + name=name, + PICS=PICS, + config=config, + steps=steps, + type=type, + path=path, + ) + + +def test_python_test_name() -> None: + """Test that test name is set as description in metadata.""" + name = "Another Test Name" + test = python_test_instance(name=name) + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.metadata["description"] == name + + +def test_python_test_python_version() -> None: + """Test that test case python version is set correctly in class factory.""" + test = python_test_instance() + python_test_version = "best_version" + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version=python_test_version + ) + assert case_class.python_test_version == python_test_version + + +def test_python_test_python() -> None: + """Test that test python_test property is as expected in subclass of PythonTestCase.""" + test = python_test_instance() + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.python_test is test + + +def test_python_test_case_class_pics() -> None: + """Test that the PICS of the python test is available in the class method PICS on + TestCase.""" + test_PICS = set(["PICS.D", "PICS.C"]) + test = python_test_instance(PICS=test_PICS) + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.pics() == test_PICS + + +def test_python_test_case_class_default_test_parameters() -> None: + """Test that the default_test_parameters of the python test is available in the class + method default_test_parameters on TestCase. + + Also parameters with type in Python test should be flattened and type dropped.""" + + test_input_config = { + "param1": "value1", + "param2": {"type": "config_type", "defaultValue": "value2"}, + } + + test = python_test_instance(config=test_input_config) + expected_default_test_parameters = {"param1": "value1", "param2": "value2"} + + # Create a subclass of PythonTest + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.default_test_parameters() == expected_default_test_parameters + + +def test_automated_test_case_class_factory_subclass_mapping() -> None: + """Test Automated tests are created as a subclass of + PythonChipToolTestCase.""" + test = python_test_instance(type=PythonTestType.AUTOMATED) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert issubclass(case_class, PythonChipToolTestCase) + + +def test_class_factory_test_public_id() -> None: + """Test that class factory correctly finds identifier 'TC-XX-1.1' in python test name. + And set it as public_id in metadata""" + test_data = [ + {"name": "TC-AB-1.2", "public_id": "TC-AB-1.2"}, + {"name": "[TC-CD-3.4]", "public_id": "TC-CD-3.4"}, + {"name": "Test Name before [TC-EF-5.6]", "public_id": "TC-EF-5.6"}, + {"name": "[TC-GH-7.8] Test Name after", "public_id": "TC-GH-7.8"}, + {"name": "Before and [TC-IJ-9.0] after", "public_id": "TC-IJ-9.0"}, + {"name": "Before and TC-KL-10.11 after", "public_id": "TC-KL-10.11"}, + {"name": "TC-MORE-NAME-13.110", "public_id": "TC-MORE-NAME-13.110"}, + ] + for data in test_data: + test = python_test_instance(name=data["name"]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.metadata["public_id"] == data["public_id"] + + +def test_class_factory_test_class_name() -> None: + """Test that class factory correctly finds identifier 'TC-XX-1.1', convert it to + a safe class name, eg TC_XX_1_1""" + test_data = [ + {"name": "TC-AB-1.2", "class_name": "TC_AB_1_2"}, + {"name": "[TC-CD-3.4]", "class_name": "TC_CD_3_4"}, + {"name": "Test Name before [TC-EF-5.6]", "class_name": "TC_EF_5_6"}, + {"name": "[TC-GH-7.8] Test Name after", "class_name": "TC_GH_7_8"}, + {"name": "Before and [TC-IJ-9.0] after", "class_name": "TC_IJ_9_0"}, + {"name": "Before and TC-KL-10.11 after", "class_name": "TC_KL_10_11"}, + {"name": "TC-MORE-NAME-13.110", "class_name": "TC_MORE_NAME_13_110"}, + ] + for data in test_data: + test = python_test_instance(name=data["name"]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert case_class.__name__ == data["class_name"] + + +def test_test_type_for_automated_tests() -> None: + """Test that automated tests are set to use chip-tool""" + test = python_test_instance(type=PythonTestType.AUTOMATED) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + assert issubclass(case_class, PythonChipToolTestCase) + instance = case_class(TestCaseExecution()) + assert instance.test_type == ChipToolTestType.PYTHON_TEST + + +@pytest.mark.asyncio +async def test_python_version_logging() -> None: + """Test that all Python tests will log Python test version to test_engine_logger. + + Note that since `chip-tool` is not setup, we except the TestError raised. + """ + for type in list(PythonTestType): + test = python_test_instance(type=type) + test_python_version = "PythonVersionTest" + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version=test_python_version + ) + instance = case_class(TestCaseExecution()) + + with mock.patch.object( + target=test_engine_logger, attribute="info" + ) as logger_info: + try: + await instance.setup() + except TestError: + pass + logger_info.assert_called() + logger_info.assert_any_call(f"Python Test Version: {test_python_version}") + + +def test_default_first_steps_for_python_chip_tool_test_case() -> None: + test = python_test_instance(type=PythonTestType.AUTOMATED, steps=[]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + assert len(instance.test_steps) == 1 + assert instance.test_steps[0].name == "Start Python test" + + +def test_normal_steps_for_non_manual_tests() -> None: + """Test that non-manual tests include enabled steps.""" + for type in list(PythonTestType): + test_step = PythonTestStep(label="Step1") + test = python_test_instance(type=type, steps=[test_step]) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + # Assert normal step is present + assert len(instance.test_steps) >= 1 + assert any(s.name == test_step.label for s in instance.test_steps) + + +def test_multiple_steps_for_non_manual() -> None: + """Test that non-manual tests multiple enabled steps are all included.""" + for type in list(PythonTestType): + test_step = PythonTestStep(label="StepN") + no_steps = 5 + test = python_test_instance(type=type, steps=([test_step] * no_steps)) + case_class: Type[PythonTestCase] = PythonTestCase.class_factory( + test=test, python_test_version="version" + ) + instance = case_class(TestCaseExecution()) + + # Assert all steps from python test are added + assert len(instance.test_steps) >= no_steps + steps_from_python = [s for s in instance.test_steps if s.name == test_step.label] + assert len(steps_from_python) == no_steps + + +@pytest.mark.asyncio +async def test_setup_super_error_handling() -> None: + # ignore requirement to create_tests on init + with mock.patch("app.test_engine.models.test_case.TestCase.create_test_steps") as _: + test = PythonTestCase(TestCaseExecution()) + test.python_test_version = "some version" + # Assert this doesn't raise an exception + await test.setup() diff --git a/app/tests/python_tests/test_python_test_declarations.py b/app/tests/python_tests/test_python_test_declarations.py new file mode 100644 index 00000000..7d70b603 --- /dev/null +++ b/app/tests/python_tests/test_python_test_declarations.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from unittest import mock + +from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( + PythonCaseDeclaration, + PythonSuiteDeclaration, +) +from test_collections.sdk_tests.support.python_testing.models.test_suite import SuiteType +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTest, +) + + +def test_python_suite_declaration() -> None: + name = "TestName" + type = SuiteType.AUTOMATED + version = "SomeVersionStr" + + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_suite.PythonTestSuite.class_factory" + ) as class_factory, mock.patch( + "app.test_engine.models.test_declarations.TestSuiteDeclaration.__init__" + ) as declaration_init: + PythonSuiteDeclaration(name=name, suite_type=type, version=version) + class_factory.assert_called_once_with( + name=name, suite_type=type, python_test_version=version + ) + declaration_init.assert_called_once() + + +def test_python_case_declaration() -> None: + test = PythonTest(name="TestTest", config={}, steps=[]) + version = "SomeVersionStr" + with mock.patch( + "test_collections.sdk_tests.support.python_testing.models.test_case.PythonTestCase.class_factory" + ) as class_factory, mock.patch( + "app.test_engine.models.test_declarations.TestCaseDeclaration.__init__" + ) as declaration_init: + PythonCaseDeclaration(test=test, python_test_version=version) + class_factory.assert_called_once_with(test=test, python_test_version=version) + declaration_init.assert_called_once() diff --git a/app/tests/python_tests/test_sdk_python_collection.py b/app/tests/python_tests/test_sdk_python_collection.py new file mode 100644 index 00000000..367a3bf9 --- /dev/null +++ b/app/tests/python_tests/test_sdk_python_collection.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +import pytest + +from test_collections.sdk_tests.support.python_testing.models.python_test_folder import ( + PythonTestFolder, +) +from test_collections.sdk_tests.support.python_testing.models.python_test_models import ( + PythonTestType, +) +from test_collections.sdk_tests.support.python_testing.models.test_declarations import ( + PythonCaseDeclaration, + PythonCollectionDeclaration, +) +from test_collections.sdk_tests.support.python_testing.sdk_python_test_tests import ( + sdk_python_test_collection, +) + +VERSION_FILE_FILENAME = ".version" +VERSION_FILE_PATH = Path("/app/backend/test_collections/sdk_tests/sdk_checkout/") + + +@pytest.fixture +def python_test_collection() -> PythonCollectionDeclaration: + test_sdk_python_path = Path(__file__).parent / "test_python_script" + folder = PythonTestFolder( + path=test_sdk_python_path, filename_pattern="UnitTest_TC_*" + ) + return sdk_python_test_collection(folder) + + +def test_sdk_python_test_collection( + python_test_collection: PythonCollectionDeclaration, +) -> None: + assert python_test_collection.name == "SDK Python Tests" + assert len(python_test_collection.test_suites.keys()) == 1 + + # test version number + test_sdk_python_version_path = VERSION_FILE_PATH / VERSION_FILE_FILENAME + with open(test_sdk_python_version_path, "r") as version_file: + assert ( + python_test_collection.python_test_version == version_file.read().rstrip() + ) + + +def test_automated_suite(python_test_collection: PythonCollectionDeclaration) -> None: + expected_automated_test_cases = 1 + + # Assert automated tests cases + assert "Python Testing Suite" in python_test_collection.test_suites.keys() + automated_suite = python_test_collection.test_suites["Python Testing Suite"] + assert len(automated_suite.test_cases) == expected_automated_test_cases + + type_count = dict.fromkeys(PythonTestType, 0) + for test_case in automated_suite.test_cases.values(): + assert isinstance(test_case, PythonCaseDeclaration) + type_count[test_case.test_type] += 1 + + assert type_count[PythonTestType.AUTOMATED] == expected_automated_test_cases diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py index 4ca11a15..e639f7d1 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_models.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_models.py @@ -41,6 +41,6 @@ class PythonTest(BaseModel): name: str PICS: set[str] = set() config: dict[str, Any] - steps: list[PythonTestStep] = Field(alias="tests") + steps: list[PythonTestStep] type: PythonTestType = PythonTestType.AUTOMATED path: Optional[Path] diff --git a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py index 25d6445c..4f22227f 100644 --- a/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/sdk_tests/support/python_testing/models/python_test_parser.py @@ -34,21 +34,21 @@ def parse_python_test(path: Path) -> PythonTest: This will also annotate parsed python test with it's path and test type. """ - python_steps: list[PythonTestStep] = [] + tc_steps: list[PythonTestStep] = [] # Currently PICS and config is not configured in Python Testing tc_pics: list = [] tc_config: dict = {} - tc_desc, python_steps = __extract_tcs_info(path) + tc_desc, tc_steps = __extract_tcs_info(path) - if not tc_desc or not python_steps: + if not tc_desc or not tc_steps: # The file name from path tc_name = path.name.split(".")[0] raise PythonParserException( f"Test Case {tc_name} does not have methods desc_{tc_name} or steps_{tc_name}" ) - test = PythonTest(name=tc_desc, tests=python_steps, config=tc_config, PICS=tc_pics) + test = PythonTest(name=tc_desc, steps=tc_steps, config=tc_config, PICS=tc_pics) test.path = path test.type = PythonTestType.AUTOMATED @@ -60,8 +60,9 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: parsed_python_file = ast.parse(python_file.read()) classes = [c for c in parsed_python_file.body if isinstance(c, ast.ClassDef)] + # Get TC description and TC steps from python test file tc_desc: str = "" - python_steps: List[PythonTestStep] = [] + tc_steps: List[PythonTestStep] = [] for class_ in classes: methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)] @@ -69,9 +70,9 @@ def __extract_tcs_info(path: Path) -> Tuple[str, List[PythonTestStep]]: if "desc_" in method.name: tc_desc = method.body[0].value.value elif "steps_" in method.name: - python_steps = __retrieve_steps(method) + tc_steps = __retrieve_steps(method) - return tc_desc, python_steps + return tc_desc, tc_steps def __retrieve_steps(method: ast.FunctionDef) -> List[PythonTestStep]: diff --git a/test_collections/sdk_tests/support/python_testing/models/test_case.py b/test_collections/sdk_tests/support/python_testing/models/test_case.py index a45fd75d..a1da6185 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_case.py @@ -17,16 +17,11 @@ from typing import Any, Type, TypeVar from app.chip_tool.chip_tool import ChipToolTestType -from app.chip_tool.test_case import ChipToolManualPromptTest, ChipToolTest +from app.chip_tool.test_case import ChipToolTest from app.test_engine.logger import test_engine_logger -from app.test_engine.models import ( - ManualTestCase, - ManualVerificationTestStep, - TestCase, - TestStep, -) +from app.test_engine.models import TestCase, TestStep -from .python_test_models import PythonTest, PythonTestStep, PythonTestType +from .python_test_models import PythonTest # Custom type variable used to annotate the factory method in PythonTestCase. T = TypeVar("T", bound="PythonTestCase") @@ -120,17 +115,6 @@ def __class_name(identifier: str) -> str: """Replace all non-alphanumeric characters with _ to make valid class name.""" return re.sub("[^0-9a-zA-Z]+", "_", identifier) - def _append_automated_test_step(self, python_test_step: PythonTestStep) -> None: - """ - Disabled steps are ignored. - (Such tests will be marked as 'Steps Disabled' elsewhere) - - UserPrompt are special cases that will prompt test operator for input. - """ - - step = TestStep(python_test_step.label) - self.test_steps.append(step) - class PythonChipToolTestCase(PythonTestCase, ChipToolTest): """Automated Python test cases.""" @@ -140,4 +124,5 @@ class PythonChipToolTestCase(PythonTestCase, ChipToolTest): def create_test_steps(self) -> None: self.test_steps = [TestStep("Start Python test")] for step in self.python_test.steps: - self._append_automated_test_step(step) + python_test_step = TestStep(step.label) + self.test_steps.append(python_test_step) diff --git a/test_collections/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/sdk_tests/support/python_testing/models/test_suite.py index 8ababa0b..5b13e4a6 100644 --- a/test_collections/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/sdk_tests/support/python_testing/models/test_suite.py @@ -53,10 +53,7 @@ def class_factory( cls, suite_type: SuiteType, name: str, python_test_version: str ) -> Type[T]: """Dynamically declares a subclass based on the type of test suite.""" - suite_class = PythonTestSuite - - if suite_type == SuiteType.AUTOMATED: - suite_class = ChipToolPythonTestSuite + suite_class = ChipToolPythonTestSuite return suite_class.__class_factory( name=name, python_test_version=python_test_version diff --git a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py index 3e91b76f..c16d10d4 100644 --- a/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py +++ b/test_collections/sdk_tests/support/python_testing/sdk_python_test_tests.py @@ -28,12 +28,12 @@ ### # This file hosts logic load and parse Python test-cases, located in -# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. The `sdk` sub-folder here is automatically -# maintained using the `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. +# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`. +# The `sdk` sub-folder here is automatically maintained using the +# `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script. # -# The Python Tests are organized into 2 Test Suites: -# - Automated and Semi-Automated using Chip-Tool -# - Simulated using Chip-App1 +# The Python Tests are organized into 1 Test Suite: +# - Automated ### SDK_PYTHON_TEST_PATH = Path( @@ -68,11 +68,7 @@ def _parse_python_test_to_test_case_declaration( def _parse_all_sdk_python_tests( python_test_files: list[Path], python_test_version: str ) -> list[PythonSuiteDeclaration]: - """Parse all python test files and organize them in the 3 test suites: - - Automated and Semi-Automated using Chip-Tool - - Simulated using Chip-App1 - - Manual - """ + """Parse all python test files and add them into Automated Suite""" suites = _init_test_suites(python_test_version) for python_test_file in python_test_files: @@ -85,7 +81,7 @@ def _parse_all_sdk_python_tests( suites[SuiteType.AUTOMATED].add_test_case(test_case) except PythonParserException as e: # If an exception was raised during parse process, the python file will be - # ignored and the loop will continue with the next python file + # ignored and the loop will continue with the next file logger.error( f"Error while parsing Python File: {python_test_file} \nError:{e}" ) @@ -96,7 +92,7 @@ def _parse_all_sdk_python_tests( def sdk_python_test_collection( python_test_folder: PythonTestFolder = SDK_PYTHON_TEST_FOLDER, ) -> PythonCollectionDeclaration: - """Declare a new collection of test suites with the 3 test suites.""" + """Declare a new collection of test suites.""" collection = PythonCollectionDeclaration( name="SDK Python Tests", folder=python_test_folder )