[Redesign/Refactoring] Move chiptool-specific code under test_collect…

…ions (#24)

* Move chiptool-specific code under test_collections

* Move chiptool test code under test_collections

* Fix flake8 issue

* Use SDK_CHECKOUT_PATH variable

* Update import

* Fix black failure

* Update python tests submodule

* Move python tests unit tests to test_collections

* Fix import paths and merge issues

* Update python tests submodule

.coveragerc

@@ -1,2 +1,4 @@
 [run]
-omit = */app/tests/*
+omit =
+        */app/tests/*
+        */test_collections/sdk_tests/support/tests/*

.vscode/settings.json

@@ -35,6 +35,7 @@
     "python.testing.pytestPath": "/usr/local/bin/pytest",
     "python.testing.pytestArgs": [
         "app/tests",
+        "test_collections/sdk_tests/support/tests",
         "--no-cov"
     ],
     "python.linting.ignorePatterns": [

scripts/test-local.sh

@@ -2,43 +2,39 @@
 set -e
 set -x
 
- #
- # Copyright (c) 2023 Project CHIP Authors
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
-
-
+#
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 #!/bin/sh
-for arg in "$@"
-do
+for arg in "$@"; do
     case $arg in
-        --run-platform-dependant)
+    --run-platform-dependant)
         RUN_ALL_TESTS=1
         shift # Remove --run-all from processing
         ;;
-        *)
+    *)
         OTHER_ARGUMENTS+=("$1")
         shift # Remove generic argument from processing
         ;;
     esac
 done
 
-if [[ $RUN_ALL_TESTS -eq 1 ]]
-then
+if [[ $RUN_ALL_TESTS -eq 1 ]]; then
     echo "Running all tests"
-    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing app/tests "${@}"
+    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing app/tests test_collections/sdk_tests/support/tests "${@}"
 else
     echo "Skipping platform dependant tests"
-    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing --ignore=app/tests/platform_dependent_tests app/tests "${@}"
+    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing --ignore=app/tests/platform_dependent_tests app/tests test_collections/sdk_tests/support/tests "${@}"
 fi

app/chip_tool/chip_tool.py → test_collections/sdk_tests/support/chip_tool/chip_tool.py

@@ -45,6 +45,7 @@
 from app.singleton import Singleton
 from app.test_engine.logger import CHIP_LOG_FORMAT, CHIPTOOL_LEVEL
 from app.test_engine.logger import test_engine_logger as logger
+from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH
 
 from .exec_run_in_container import ExecResultExtended, exec_run_in_container
 
@@ -98,15 +99,10 @@
 DOCKER_CREDENTIALS_DEVELOPMENT_PATH = "/credentials/development"
 
 # Websocket runner
-BACKEND_ROOT = Path(__file__).parents[2]
-TEST_COLLECTION_SDK_CHECKOUT_PATH = BACKEND_ROOT / Path(
-    "test_collections/sdk_tests/sdk_checkout/"
-)
-YAML_TESTS_PATH_BASE = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path("yaml_tests/")
+YAML_TESTS_PATH_BASE = SDK_CHECKOUT_PATH / Path("yaml_tests/")
 YAML_TESTS_PATH = YAML_TESTS_PATH_BASE / Path("yaml/sdk")
-XML_SPEC_DEFINITION_PATH = TEST_COLLECTION_SDK_CHECKOUT_PATH / Path(
-    "sdk_runner/specifications/chip/"
-)
+XML_SPEC_DEFINITION_PATH = SDK_CHECKOUT_PATH / Path("sdk_runner/specifications/chip/")
+
 # Python Testing Folder
 LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections"
 LOCAL_PYTHON_TESTING_PATH = Path(

app/chip_tool/test_case.py → test_collections/sdk_tests/support/chip_tool/test_case.py

@@ -22,8 +22,6 @@
 from matter_yamltests.hooks import TestParserHooks, TestRunnerHooks
 from matter_yamltests.parser import PostProcessResponseResult, TestStep
 
-from app.chip_tool import ChipTool
-from app.chip_tool.chip_tool import ChipToolTestType
 from app.models import TestStateEnum
 from app.models.test_case_execution import TestCaseExecution
 from app.test_engine.logger import CHIP_LOG_FORMAT, CHIPTOOL_LEVEL, test_engine_logger
@@ -36,6 +34,8 @@
 from app.user_prompt_support.uploaded_file_support import UploadFile
 from app.user_prompt_support.user_prompt_manager import user_prompt_manager
 from app.user_prompt_support.user_prompt_support import UserPromptSupport
+from test_collections.sdk_tests.support.chip_tool import ChipTool
+from test_collections.sdk_tests.support.chip_tool.chip_tool import ChipToolTestType
 
 CHIP_TOOL_DEFAULT_PROMPT_TIMEOUT_S = 60  # seconds
 OUTCOME_TIMEOUT_S = 60 * 10  # Seconds

app/chip_tool/test_suite.py → test_collections/sdk_tests/support/chip_tool/test_suite.py

@@ -15,9 +15,6 @@
 #
 from typing import Optional
 
-from app.chip_tool import ChipTool
-from app.chip_tool.chip_tool import ChipToolTestType
-from app.chip_tool.test_case import PromptOption
 from app.models import TestSuiteExecution
 from app.otbr_manager.otbr_manager import ThreadBorderRouter
 from app.schemas.test_environment_config import (
@@ -29,6 +26,9 @@
 from app.test_engine.models import TestSuite
 from app.user_prompt_support.prompt_request import OptionsSelectPromptRequest
 from app.user_prompt_support.user_prompt_support import UserPromptSupport
+from test_collections.sdk_tests.support.chip_tool import ChipTool
+from test_collections.sdk_tests.support.chip_tool.chip_tool import ChipToolTestType
+from test_collections.sdk_tests.support.chip_tool.test_case import PromptOption
 
 CHIP_APP_PAIRING_CODE = "CHIP:SVR: Manual pairing code:"
 

test_collections/sdk_tests/support/python_testing/models/test_case.py

@@ -18,10 +18,13 @@
 from multiprocessing.managers import BaseManager
 from typing import Any, Type, TypeVar
 
-from app.chip_tool.chip_tool import PICS_FILE_PATH, ChipTool
 from app.models import TestCaseExecution
 from app.test_engine.logger import test_engine_logger as logger
 from app.test_engine.models import TestCase, TestStep
+from test_collections.sdk_tests.support.chip_tool.chip_tool import (
+    PICS_FILE_PATH,
+    ChipTool,
+)
 
 from .python_test_models import PythonTest
 from .python_testing_hooks_proxy import (

test_collections/sdk_tests/support/python_testing/models/test_suite.py

@@ -16,9 +16,9 @@
 from enum import Enum
 from typing import Type, TypeVar
 
-from app.chip_tool import ChipTool
 from app.test_engine.logger import test_engine_logger as logger
 from app.test_engine.models import TestSuite
+from test_collections.sdk_tests.support.chip_tool import ChipTool
 
 
 class SuiteType(Enum):

app/tests/chip_tool/test_chip_tool.py → test_collections/sdk_tests/support/tests/chip_tool/test_chip_tool.py

@@ -25,8 +25,13 @@
 from matter_yamltests.parser_builder import TestParserBuilderConfig
 from matter_yamltests.runner import TestRunnerConfig
 
-from app.chip_tool import ChipTool
-from app.chip_tool.chip_tool import (
+from app.container_manager import container_manager
+from app.core.config import settings
+from app.schemas.pics import PICSError
+from app.tests.utils.docker import make_fake_container
+from app.tests.utils.test_pics_data import create_random_pics
+from test_collections.sdk_tests.support.chip_tool import ChipTool
+from test_collections.sdk_tests.support.chip_tool.chip_tool import (
     CHIP_APP_EXE,
     CHIP_TOOL_ARG_PAA_CERTS_PATH,
     CHIP_TOOL_CONTINUE_ON_FAILURE_VALUE,
@@ -42,12 +47,9 @@
     ChipToolTestType,
     ChipToolUnknownTestType,
 )
-from app.chip_tool.exec_run_in_container import ExecResultExtended
-from app.container_manager import container_manager
-from app.core.config import settings
-from app.schemas.pics import PICSError
-from app.tests.utils.docker import make_fake_container
-from app.tests.utils.test_pics_data import create_random_pics
+from test_collections.sdk_tests.support.chip_tool.exec_run_in_container import (
+    ExecResultExtended,
+)
 
 
 @pytest.mark.asyncio
@@ -63,7 +65,8 @@ async def test_start_container() -> None:
     ), mock.patch.object(
         target=container_manager, attribute="get_container", return_value=None
     ), mock.patch(
-        target="app.chip_tool.chip_tool.backend_container"
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool."
+        "backend_container"
     ), mock.patch.object(
         target=container_manager, attribute="create_container"
     ) as mock_create_container, mock.patch.object(
@@ -92,7 +95,8 @@ async def test_start_container_using_paa_certs() -> None:
     ), mock.patch.object(
         target=container_manager, attribute="get_container", return_value=None
     ), mock.patch(
-        target="app.chip_tool.chip_tool.backend_container"
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool."
+        "backend_container"
     ), mock.patch.object(
         target=container_manager, attribute="create_container"
     ) as mock_create_container, mock.patch.object(
@@ -392,7 +396,8 @@ async def test_destroy_container_once() -> None:
     ), mock.patch.object(
         target=container_manager, attribute="get_container", return_value=None
     ), mock.patch(
-        target="app.chip_tool.chip_tool.backend_container"
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool."
+        "backend_container"
     ), mock.patch.object(
         target=container_manager, attribute="destroy"
     ) as mock_destroy, mock.patch.object(
@@ -448,7 +453,7 @@ async def test_set_pics() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.subprocess.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool.subprocess.run",
         return_value=CompletedProcess(expected_command, 0),
     ) as mock_run:
         await chip_tool.start_server(test_type)
@@ -469,7 +474,7 @@ def test_set_pics_with_error() -> None:
     pics = create_random_pics()
 
     with mock.patch(
-        target="app.chip_tool.chip_tool.subprocess.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool.subprocess.run",
         return_value=CompletedProcess("", 1),
     ), pytest.raises(PICSError):
         chip_tool.set_pics(pics, in_container=False)
@@ -501,7 +506,8 @@ async def test_send_command_default_prefix() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.exec_run_in_container",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool."
+        "exec_run_in_container",
         return_value=mock_result,
     ) as mock_exec_run:
         await chip_tool.start_server(test_type)
@@ -544,7 +550,8 @@ async def test_send_command_custom_prefix() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.exec_run_in_container",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool."
+        "exec_run_in_container",
         return_value=mock_result,
     ) as mock_exec_run:
         await chip_tool.start_server(test_type)
@@ -585,10 +592,12 @@ async def test_run_test_default_config() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.start",
         return_value=True,
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.run",
         return_value=True,
     ) as mock_run:
         await chip_tool.start_server(test_type)
@@ -642,10 +651,12 @@ async def test_run_test_custom_timeout() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.start",
         return_value=True,
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.run",
         return_value=True,
     ) as mock_run:
         await chip_tool.start_server(test_type)
@@ -693,10 +704,12 @@ async def test_run_test_with_custom_parameter() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.start",
         return_value=True,
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.run",
         return_value=True,
     ) as mock_run:
         await chip_tool.start_server(test_type)
@@ -745,10 +758,12 @@ async def test_run_test_with_endpoint_parameter() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.start",
         return_value=True,
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.run",
         return_value=True,
     ) as mock_run:
         await chip_tool.start_server(test_type)
@@ -796,10 +811,12 @@ async def test_run_test_with_nodeID_and_cluster_parameters() -> None:
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.start",
         return_value=True,
     ), mock.patch(
-        target="app.chip_tool.chip_tool.WebSocketRunner.run",
+        target="test_collections.sdk_tests.support.chip_tool.chip_tool"
+        ".WebSocketRunner.run",
         return_value=True,
     ) as mock_run:
         await chip_tool.start_server(test_type)

app/tests/chip_tool/test_docker_extension.py → test_collections/sdk_tests/support/tests/chip_tool/test_docker_extension.py

@@ -15,8 +15,10 @@
 #
 from typing import Generator
 
-from app.chip_tool.exec_run_in_container import exec_run_in_container
 from app.tests.utils.docker import make_fake_container
+from test_collections.sdk_tests.support.chip_tool.exec_run_in_container import (
+    exec_run_in_container,
+)
 
 
 def test_exec_run_in_container_not_stream() -> None:

app/tests/chip_tool/test_test_suite.py → test_collections/sdk_tests/support/tests/chip_tool/test_test_suite.py

@@ -20,15 +20,15 @@
 
 import pytest
 
-from app.chip_tool.test_case import PromptOption
-from app.chip_tool.test_suite import (
+from app.models.test_suite_execution import TestSuiteExecution
+from app.user_prompt_support.constants import UserResponseStatusEnum
+from app.user_prompt_support.prompt_response import PromptResponse
+from test_collections.sdk_tests.support.chip_tool.test_case import PromptOption
+from test_collections.sdk_tests.support.chip_tool.test_suite import (
     ChipToolSuite,
     DUTCommissioningError,
     SuiteSetupError,
 )
-from app.models.test_suite_execution import TestSuiteExecution
-from app.user_prompt_support.constants import UserResponseStatusEnum
-from app.user_prompt_support.prompt_response import PromptResponse
 
 RETRY_PROMPT_RESPONSE = PromptResponse(
     response=PromptOption.RETRY, status_code=UserResponseStatusEnum.OKAY

app/tests/python_tests/test_python_test_case.py → test_collections/sdk_tests/support/tests/python_tests/test_python_test_case.py

@@ -21,10 +21,9 @@
 
 import pytest
 
-from app.chip_tool.chip_tool import ChipToolTestType
-from app.chip_tool.test_case import TestError
 from app.models.test_case_execution import TestCaseExecution
 from app.test_engine.logger import test_engine_logger
+from test_collections.sdk_tests.support.chip_tool.test_case import TestError
 from test_collections.sdk_tests.support.models.matter_test_models import (
     MatterTestStep,
     MatterTestType,