Merge branch 'project-chip:main' into main

.coveragerc

@@ -1,2 +1,4 @@
 [run]
-omit = */app/tests/*
+omit =
+        */app/tests/*
+        */test_collections/sdk_tests/support/tests/*

.vscode/settings.json

@@ -35,6 +35,7 @@
     "python.testing.pytestPath": "/usr/local/bin/pytest",
     "python.testing.pytestArgs": [
         "app/tests",
+        "test_collections/sdk_tests/support/tests",
         "--no-cov"
     ],
     "python.linting.ignorePatterns": [

app/core/config.py

@@ -115,7 +115,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool:
     SDK_DOCKER_IMAGE: str = "connectedhomeip/chip-cert-bins"
     SDK_DOCKER_TAG: str = "f06d9520d02d68076c5accbf839f168cda89c47c"
     # SDK SHA: used to fetch test YAML from SDK.
-    SDK_SHA: str = "4fde331ac902b7349653f069e9da6c933efa1466"
+    SDK_SHA: str = "696975f5d89f1d4568bb5d3f625343fa1200abad"
 
     class Config:
         case_sensitive = True

app/test_engine/logger.py

@@ -22,5 +22,9 @@
 CHIPTOOL_LEVEL = "CHIPTOOL"
 test_engine_logger.level(CHIPTOOL_LEVEL, no=21, icon="🤖", color="<cyan>")
 
+# Add custom logger for python tests
+PYTHON_TEST_LEVEL = "PYTHON_TEST"
+test_engine_logger.level(PYTHON_TEST_LEVEL, no=22, icon="🐍", color="<cyan>")
+
 # Log format for logs that come from CHIP. Arguments: module (e.g. "DMG"), message
 CHIP_LOG_FORMAT = "CHIP:{} {}"

scripts/test-local.sh

@@ -2,43 +2,39 @@
 set -e
 set -x
 
- #
- # Copyright (c) 2023 Project CHIP Authors
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
-
-
+#
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 #!/bin/sh
-for arg in "$@"
-do
+for arg in "$@"; do
     case $arg in
-        --run-platform-dependant)
+    --run-platform-dependant)
         RUN_ALL_TESTS=1
         shift # Remove --run-all from processing
         ;;
-        *)
+    *)
         OTHER_ARGUMENTS+=("$1")
         shift # Remove generic argument from processing
         ;;
     esac
 done
 
-if [[ $RUN_ALL_TESTS -eq 1 ]]
-then
+if [[ $RUN_ALL_TESTS -eq 1 ]]; then
     echo "Running all tests"
-    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing app/tests "${@}"
+    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing app/tests test_collections/sdk_tests/support/tests "${@}"
 else
     echo "Skipping platform dependant tests"
-    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing --ignore=app/tests/platform_dependent_tests app/tests "${@}"
+    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing --ignore=app/tests/platform_dependent_tests app/tests test_collections/sdk_tests/support/tests "${@}"
 fi

app/chip_tool/chip_tool.py → test_collections/sdk_tests/support/chip_tool/chip_tool.py

@@ -45,6 +45,7 @@
 from app.singleton import Singleton
 from app.test_engine.logger import CHIP_LOG_FORMAT, CHIPTOOL_LEVEL
 from app.test_engine.logger import test_engine_logger as logger
+from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH
 
 from .exec_run_in_container import ExecResultExtended, exec_run_in_container
 
@@ -98,15 +99,10 @@
 DOCKER_CREDENTIALS_DEVELOPMENT_PATH = "/credentials/development"
 
 # Websocket runner
-BACKEND_ROOT = Path(__file__).parents[2]
-TEST_COLLECTION_SDK_CHECKOUT_PATH = BACKEND_ROOT / Path(
-    "test_collections/sdk_tests/sdk_checkout/"
-)
-YAML_TESTS_PATH_BASE = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path("yaml_tests/")
+YAML_TESTS_PATH_BASE = SDK_CHECKOUT_PATH / Path("yaml_tests/")
 YAML_TESTS_PATH = YAML_TESTS_PATH_BASE / Path("yaml/sdk")
-XML_SPEC_DEFINITION_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path(
-    "sdk_runner/specifications/chip/"
-)
+XML_SPEC_DEFINITION_PATH = SDK_CHECKOUT_PATH / Path("sdk_runner/specifications/chip/")
+
 # Python Testing Folder
 LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections"
 LOCAL_PYTHON_TESTING_PATH = Path(

app/chip_tool/test_case.py → test_collections/sdk_tests/support/chip_tool/test_case.py

@@ -22,8 +22,6 @@
 from matter_yamltests.hooks import TestParserHooks, TestRunnerHooks
 from matter_yamltests.parser import PostProcessResponseResult, TestStep
 
-from app.chip_tool import ChipTool
-from app.chip_tool.chip_tool import ChipToolTestType
 from app.models import TestStateEnum
 from app.models.test_case_execution import TestCaseExecution
 from app.test_engine.logger import CHIP_LOG_FORMAT, CHIPTOOL_LEVEL, test_engine_logger
@@ -36,6 +34,8 @@
 from app.user_prompt_support.uploaded_file_support import UploadFile
 from app.user_prompt_support.user_prompt_manager import user_prompt_manager
 from app.user_prompt_support.user_prompt_support import UserPromptSupport
+from test_collections.sdk_tests.support.chip_tool import ChipTool
+from test_collections.sdk_tests.support.chip_tool.chip_tool import ChipToolTestType
 
 CHIP_TOOL_DEFAULT_PROMPT_TIMEOUT_S = 60  # seconds
 OUTCOME_TIMEOUT_S = 60 * 10  # Seconds

app/chip_tool/test_suite.py → test_collections/sdk_tests/support/chip_tool/test_suite.py

@@ -15,9 +15,6 @@
 #
 from typing import Optional
 
-from app.chip_tool import ChipTool
-from app.chip_tool.chip_tool import ChipToolTestType
-from app.chip_tool.test_case import PromptOption
 from app.models import TestSuiteExecution
 from app.otbr_manager.otbr_manager import ThreadBorderRouter
 from app.schemas.test_environment_config import (
@@ -29,6 +26,9 @@
 from app.test_engine.models import TestSuite
 from app.user_prompt_support.prompt_request import OptionsSelectPromptRequest
 from app.user_prompt_support.user_prompt_support import UserPromptSupport
+from test_collections.sdk_tests.support.chip_tool import ChipTool
+from test_collections.sdk_tests.support.chip_tool.chip_tool import ChipToolTestType
+from test_collections.sdk_tests.support.chip_tool.test_case import PromptOption
 
 CHIP_APP_PAIRING_CODE = "CHIP:SVR: Manual pairing code:"
 

test_collections/sdk_tests/support/python_testing/models/python_test_parser.py

@@ -15,7 +15,7 @@
 #
 import ast
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
 from test_collections.sdk_tests.support.models.matter_test_models import (
     MatterTestStep,
@@ -73,23 +73,25 @@ def __parse_test_case_from_class(
     pics_method_name = "pics_" + tc_name
 
     methods = [m for m in class_.body if isinstance(m, ast.FunctionDef)]
-    try:
-        desc_method = next(m for m in methods if desc_method_name in m.name)
+
+    tc_desc = tc_name
+    tc_steps = []
+    tc_pics = []
+
+    desc_method = __get_method_by_name(desc_method_name, methods)
+    if desc_method:
         tc_desc = desc_method.body[BODY_INDEX].value.value  # type: ignore
 
-        steps_method = next(m for m in methods if steps_method_name in m.name)
+    # If the python test does not implement the steps template method,
+    # the test case will be presented in UI and the whole test case will be
+    # executed as one step
+    steps_method = __get_method_by_name(steps_method_name, methods)
+    if steps_method:
         tc_steps = __retrieve_steps(steps_method)
-    except StopIteration as si:  # Raised when `next` doesn't find a matching method
-        raise PythonParserException(
-            f"{path} did not contain valid definition for {tc_name}"
-        ) from si
 
-    # PICS method is optional
-    try:
-        pics_method = next(m for m in methods if pics_method_name in m.name)
+    pics_method = __get_method_by_name(pics_method_name, methods)
+    if pics_method:
         tc_pics = __retrieve_pics(pics_method)
-    except StopIteration:  # Raised when `next` doesn't find a matching method
-        tc_pics = []
 
     return PythonTest(
         name=tc_name,
@@ -102,6 +104,12 @@ def __parse_test_case_from_class(
     )
 
 
+def __get_method_by_name(
+    name: str, methods: list[ast.FunctionDef]
+) -> Optional[ast.FunctionDef]:
+    return next((m for m in methods if name in m.name), None)
+
+
 def __retrieve_steps(method: ast.FunctionDef) -> List[MatterTestStep]:
     python_steps: List[MatterTestStep] = []
     for step in method.body[BODY_INDEX].value.elts:  # type: ignore

test_collections/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py

@@ -21,7 +21,12 @@
 import sys
 from multiprocessing.managers import BaseManager
 
-import matter_testing_support
+from matter_testing_support import (
+    CommissionDeviceTest,
+    MatterTestConfig,
+    parse_matter_test_args,
+    run_tests,
+)
 
 
 class TestRunnerHooks:
@@ -31,14 +36,17 @@ class TestRunnerHooks:
 def main() -> None:
     # Load python_testing as a module. This folder is where all python script is located
     sys.path.append("/root/python_testing")
-    test_name = sys.argv[1]
 
-    config_options = sys.argv[2:]
-    config = matter_testing_support.parse_matter_test_args(config_options)
+    test_args = sys.argv[2:]
+    config = parse_matter_test_args(test_args)
 
-    if config is None:
-        raise ValueError(f"Not a valid test id: {test_name}")
+    if sys.argv[1] == "commission":
+        commission(config)
+    else:
+        run_test(test_name=sys.argv[1], config=config)
 
+
+def run_test(test_name: str, config: MatterTestConfig) -> None:
     module = importlib.import_module(test_name)
     TestClassReference = getattr(module, test_name)
 
@@ -47,7 +55,12 @@ def main() -> None:
     manager.connect()
     test_runner_hooks = manager.TestRunnerHooks()  # shared object proxy # type: ignore
 
-    matter_testing_support.run_tests(TestClassReference, config, test_runner_hooks)
+    run_tests(TestClassReference, config, test_runner_hooks)
+
+
+def commission(config: MatterTestConfig) -> None:
+    config.commission_only = True
+    run_tests(CommissionDeviceTest, config, None)
 
 
 if __name__ == "__main__":

test_collections/sdk_tests/support/python_testing/models/test_case.py

@@ -18,25 +18,24 @@
 from multiprocessing.managers import BaseManager
 from typing import Any, Type, TypeVar
 
-from app.chip_tool.chip_tool import PICS_FILE_PATH, ChipTool
 from app.models import TestCaseExecution
 from app.test_engine.logger import test_engine_logger as logger
 from app.test_engine.models import TestCase, TestStep
+from test_collections.sdk_tests.support.chip_tool.chip_tool import (
+    PICS_FILE_PATH,
+    ChipTool,
+)
 
 from .python_test_models import PythonTest
 from .python_testing_hooks_proxy import (
     SDKPythonTestResultBase,
     SDKPythonTestRunnerHooks,
 )
+from .utils import EXECUTABLE, RUNNER_CLASS_PATH, generate_command_arguments
 
 # Custom type variable used to annotate the factory method in PythonTestCase.
 T = TypeVar("T", bound="PythonTestCase")
 
-# Command line params
-RUNNER_CLASS = "test_harness_client.py"
-RUNNER_CLASS_PATH = "/root/python_testing/"
-EXECUTABLE = "python3"
-
 
 class PythonTestCase(TestCase):
     """Base class for all Python Test based test cases.
@@ -57,6 +56,15 @@ def __init__(self, test_case_execution: TestCaseExecution) -> None:
         self.__runned = 0
         self.test_stop_called = False
 
+    def next_step(self) -> None:
+        # Python tests that don't follow the template only have the default step "Start
+        # Python test", but inside the file there can be more than one test case, so the
+        # hooks steps methods will continue to be called
+        if len(self.test_steps) == 1:
+            return
+
+        super().next_step()
+
     def start(self, count: int) -> None:
         pass
 
@@ -151,16 +159,18 @@ async def execute(self) -> None:
             manager.start()
             test_runner_hooks = manager.TestRunnerHooks()  # type: ignore
 
-            runner_class = RUNNER_CLASS_PATH + RUNNER_CLASS
-            command = (
-                f"{runner_class} {self.python_test.name}"
-                " --commissioning-method on-network --discriminator 3840 --passcode"
-                " 20202021 --storage-path /root/admin_storage.json"
-                " --paa-trust-store-path /paa-root-certs"
+            command = [f"{RUNNER_CLASS_PATH} {self.python_test.name}"]
+
+            # Generate the command argument by getting the test_parameters from
+            # project configuration
+            # comissioning method is omitted because it's handled by the test suite
+            command_arguments = generate_command_arguments(
+                config=self.config, omit_commissioning_method=True
             )
+            command.extend(command_arguments)
 
             if self.chip_tool.pics_file_created:
-                command += f" --PICS {PICS_FILE_PATH}"
+                command.append(f" --PICS {PICS_FILE_PATH}")
 
             # TODO Ignoring stream from docker execution
             self.chip_tool.send_command(

test_collections/sdk_tests/support/python_testing/models/test_suite.py

@@ -14,11 +14,18 @@
 # limitations under the License.
 #
 from enum import Enum
-from typing import Type, TypeVar
+from typing import Generator, Type, TypeVar, cast
 
-from app.chip_tool import ChipTool
 from app.test_engine.logger import test_engine_logger as logger
 from app.test_engine.models import TestSuite
+from test_collections.sdk_tests.support.chip_tool import ChipTool
+
+from .utils import (
+    EXECUTABLE,
+    RUNNER_CLASS_PATH,
+    generate_command_arguments,
+    handle_logs,
+)
 
 
 class SuiteType(Enum):
@@ -74,8 +81,25 @@ async def setup(self) -> None:
         else:
             self.chip_tool.reset_pics_state()
 
+        logger.info("Commission DUT")
+        self.commission_device()
+
     async def cleanup(self) -> None:
         logger.info("Suite Cleanup")
 
         logger.info("Stopping SDK container")
         await self.chip_tool.destroy_device()
+
+    def commission_device(self) -> None:
+        command = [f"{RUNNER_CLASS_PATH} commission"]
+        command_arguments = generate_command_arguments(config=self.config)
+        command.extend(command_arguments)
+
+        exec_result = self.chip_tool.send_command(
+            command,
+            prefix=EXECUTABLE,
+            is_stream=True,
+            is_socket=False,
+        )
+
+        handle_logs(cast(Generator, exec_result.output), logger)