Skip to content

Commit

Permalink
Implemented changes for custom python test (#167)
Browse files Browse the repository at this point in the history
* Implemented changes for custom python test

* code review

* Code review

* Code reviewd

* Fixed code violations

* Chanegs after code review

* Remove unused import
  • Loading branch information
rquidute authored and hiltonlima committed Dec 17, 2024
1 parent b8df5fe commit 9d79b81
Show file tree
Hide file tree
Showing 9 changed files with 278 additions and 102 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ SerialTests.lock
test_db_creation.lock
.sha_information
test_collections/matter/sdk_tests/sdk_checkout
performance-logs
performance-logs
test_collections/matter/sdk_tests/custom_python_tests_info.json
19 changes: 19 additions & 0 deletions app/tests/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#
# Copyright (c) 2024 Project CHIP Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# isort: split - DO NOT MOVE THIS IMPORT, this should be before any app.*
from .sdk_container_mock import mock_instance

# isort: split
32 changes: 32 additions & 0 deletions app/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
# limitations under the License.
#
import asyncio
import contextlib
import sys
from importlib import import_module
from typing import AsyncGenerator, Generator
from unittest import mock

Expand Down Expand Up @@ -123,3 +126,32 @@ def block_on_serial_marker(request: pytest.FixtureRequest) -> Generator:
test_script_manager.test_script_manager.test_collections = discover_test_collections(
disabled_collections=[]
)


@contextlib.contextmanager
def use_real_sdk_container() -> Generator:
"""Context manager to temporarily use the real SDKContainer"""
# Store the mock module
mock_module = sys.modules["test_collections.matter.sdk_tests.support.sdk_container"]

# Remove the mock from sys.modules to force reload
del sys.modules["test_collections.matter.sdk_tests.support.sdk_container"]

try:
# Import the real module
real_module = import_module(
"test_collections.matter.sdk_tests.support.sdk_container"
)
yield real_module
finally:
# Restore the mock module
sys.modules[
"test_collections.matter.sdk_tests.support.sdk_container"
] = mock_module


@pytest.fixture
def real_sdk_container() -> Generator:
"""Use the real SDKContainer in a test"""
with use_real_sdk_container() as real_module: # noqa
yield real_module.SDKContainer()
107 changes: 107 additions & 0 deletions app/tests/sdk_container_mock.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#
# Copyright (c) 2024 Project CHIP Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, Mock

# Constants from the original module
DOCKER_LOGS_PATH = "/logs"
DOCKER_PAA_CERTS_PATH = "/paa-root-certs"
LOCAL_LOGS_PATH = Path("/var/tmp")
LOCAL_PAA_CERTS_PATH = Path("/var/paa-root-certs")
LOCAL_CREDENTIALS_DEVELOPMENT_PATH = Path("/var/credentials/development")
DOCKER_CREDENTIALS_DEVELOPMENT_PATH = "/credentials/development"
LOCAL_TEST_COLLECTIONS_PATH = (
"/home/ubuntu/certification-tool/backend/test_collections/matter"
)
LOCAL_PYTHON_TESTING_PATH = Path(
LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/sdk_checkout/python_testing"
)
DOCKER_PYTHON_TESTING_PATH = "/root/python_testing"
MAPPED_DATA_MODEL_VOLUME = "mapped_data_model_volume"
DOCKER_DATA_MODEL_PATH = DOCKER_PYTHON_TESTING_PATH + "/data_model"
LOCAL_RPC_PYTHON_TESTING_PATH = Path(
LOCAL_TEST_COLLECTIONS_PATH
+ "/sdk_tests/support/python_testing/models/rpc_client/test_harness_client.py"
)
DOCKER_RPC_PYTHON_TESTING_PATH = Path(
"/root/python_testing/scripts/sdk/"
"matter_testing_infrastructure/chip/testing/test_harness_client.py"
)


# Exception classes
class SDKContainerNotRunning(Exception):
"""
Raised when we attempt to use the docker container, but it is not running
"""


class SDKContainerRetrieveExitCodeError(Exception):
"""
Raised when there's an error in the attempt to retrieve an execution's exit code
"""


# Create mock instance
mock_instance = Mock()
mock_instance.is_running.return_value = True
mock_instance.pics_file_created = False
mock_instance.send_command.return_value = Mock(exit_code=0, output="mocked output")
mock_instance.start = AsyncMock()


# Create mock SDKContainer class
class MockSDKContainer:
def __new__(cls, *args: Any, **kwargs: Any) -> "MockSDKContainer":
return mock_instance


# Store mock for access
sys.mock_sdk_container = mock_instance # type: ignore

# Create and setup mock module
mock_module = type(
"mock_module",
(),
{
"SDKContainer": MockSDKContainer,
"DOCKER_LOGS_PATH": DOCKER_LOGS_PATH,
"DOCKER_PAA_CERTS_PATH": DOCKER_PAA_CERTS_PATH,
"LOCAL_LOGS_PATH": LOCAL_LOGS_PATH,
"LOCAL_PAA_CERTS_PATH": LOCAL_PAA_CERTS_PATH,
"LOCAL_CREDENTIALS_DEVELOPMENT_PATH": LOCAL_CREDENTIALS_DEVELOPMENT_PATH,
"DOCKER_CREDENTIALS_DEVELOPMENT_PATH": DOCKER_CREDENTIALS_DEVELOPMENT_PATH,
"LOCAL_TEST_COLLECTIONS_PATH": LOCAL_TEST_COLLECTIONS_PATH,
"LOCAL_PYTHON_TESTING_PATH": LOCAL_PYTHON_TESTING_PATH,
"DOCKER_PYTHON_TESTING_PATH": DOCKER_PYTHON_TESTING_PATH,
"MAPPED_DATA_MODEL_VOLUME": MAPPED_DATA_MODEL_VOLUME,
"DOCKER_DATA_MODEL_PATH": DOCKER_DATA_MODEL_PATH,
"LOCAL_RPC_PYTHON_TESTING_PATH": LOCAL_RPC_PYTHON_TESTING_PATH,
"DOCKER_RPC_PYTHON_TESTING_PATH": DOCKER_RPC_PYTHON_TESTING_PATH,
"SDKContainerNotRunning": SDKContainerNotRunning,
"SDKContainerRetrieveExitCodeError": SDKContainerRetrieveExitCodeError,
"__file__": "mocked_path",
"__loader__": None,
"__spec__": None,
"__name__": "test_collections.matter.sdk_tests.support.sdk_container",
},
)()

# Patch the module
sys.modules["test_collections.matter.sdk_tests.support.sdk_container"] = mock_module
3 changes: 2 additions & 1 deletion cspell.json
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@
"test_collections/matter/sdk_tests/sdk_checkout",
"test_collections/matter/python_tests/docs/common_test_failures/",
"sdk_patch",
".devcontainer"
".devcontainer",
"test_collections/matter/sdk_tests/*.json"
],
"enableFiletypes": [
"shellscript"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,22 @@
SDK_TESTS_PATH = Path(__file__).parent.parent.parent
PYTHON_TESTING_PATH = SDK_TESTS_PATH / "sdk_checkout/python_testing"
JSON_OUTPUT_FILE_PATH = PYTHON_TESTING_PATH / TEST_INFO_JSON_FILENAME
COMPLETE_JSON_OUTPUT_FILE_FOLDER = SDK_TESTS_PATH
PYTHON_SCRIPTS_PATH = PYTHON_TESTING_PATH / "scripts/sdk"
PYTHON_SCRIPTS_FOLDER = SDKTestFolder(path=PYTHON_SCRIPTS_PATH, filename_pattern="TC*")

CUSTOM_PYTHON_SCRIPTS_PATH = PYTHON_TESTING_PATH / "scripts/custom"
CUSTOM_PYTHON_SCRIPTS_FOLDER = SDKTestFolder(
path=CUSTOM_PYTHON_SCRIPTS_PATH, filename_pattern="TC*"
)


PYTHON_TESTS_PARSED_FILE = SDK_TESTS_PATH / "python_tests_info.json"
CUSTOM_PYTHON_TESTS_PARSED_FILE = SDK_TESTS_PATH / "custom_python_tests_info.json"

CONTAINER_TH_CLIENT_EXEC = "python3 /root/python_testing/scripts/sdk/matter_testing_infrastructure/chip/testing/test_harness_client.py" # noqa

sdk_container: SDKContainer = SDKContainer()


def base_test_classes(module: ast.Module) -> list[ast.ClassDef]:
"""Find classes that inherit from MatterBaseTest.
Expand All @@ -59,9 +69,9 @@ def base_test_classes(module: ast.Module) -> list[ast.ClassDef]:
]


def get_command_list() -> list:
def get_command_list(test_folder: SDKTestFolder) -> list:
python_script_commands = []
python_test_files = PYTHON_SCRIPTS_FOLDER.file_paths(extension=".py")
python_test_files = test_folder.file_paths(extension=".py")
python_test_files.sort()

for python_test_file in python_test_files:
Expand All @@ -79,7 +89,10 @@ def get_command_list() -> list:
return python_script_commands


async def proccess_commands_sdk_container(commands: list) -> None:
async def proccess_commands_sdk_container(
commands: list,
json_output_file: Path,
) -> None:
complete_json = []
errors_found: list[str] = []
test_function_count = 0
Expand Down Expand Up @@ -114,20 +127,18 @@ async def proccess_commands_sdk_container(commands: list) -> None:

sdk_container.destroy()

complete_json_path = COMPLETE_JSON_OUTPUT_FILE_FOLDER / "python_tests_info.json"

# complete_json.append({"sdk_sha": matter_settings.SDK_SHA})
# Create a wrapper object with sdk_sha at root level
json_output = {"sdk_sha": matter_settings.SDK_SHA, "tests": complete_json}

with open(complete_json_path, "w") as json_file:
with open(json_output_file, "w") as json_file:
json.dump(json_output, json_file, indent=4, sort_keys=True)
json_file.close()

print("###########################################################################")
print("############################### REPORT ################################")
print("###########################################################################")
print(f">>>>>>>> Output JSON file: {complete_json_path}")
print(f">>>>>>>> Output JSON file: {json_output_file}")
print(f">>>>>>>> Total of test functions: {test_function_count}")
print(
(
Expand All @@ -143,10 +154,16 @@ async def proccess_commands_sdk_container(commands: list) -> None:
print("###########################################################################")


if __name__ == "__main__":
python_scripts_command_list = get_command_list()
loop = asyncio.get_event_loop()
loop.run_until_complete(
proccess_commands_sdk_container(python_scripts_command_list)
async def generate_python_test_json_file(
test_folder: SDKTestFolder = PYTHON_SCRIPTS_FOLDER,
json_output_file: Path = PYTHON_TESTS_PARSED_FILE,
) -> None:
python_scripts_command_list = get_command_list(test_folder=test_folder)

await proccess_commands_sdk_container(
python_scripts_command_list, json_output_file=json_output_file
)
loop.close()


if __name__ == "__main__":
asyncio.run(generate_python_test_json_file())
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,13 @@ def parse_python_script(path: Path) -> list[PythonTest]:
Example: file TC_ACE_1_3.py has the methods test_TC_ACE_1_3, desc_TC_ACE_1_3,
pics_TC_ACE_1_3 and steps_TC_ACE_1_3.
"""
python_tests: list[PythonTest] = []

with open(path, "r") as json_file:
parsed_scripts = json.load(json_file)

python_tests: list[PythonTest] = []
if len(parsed_scripts) == 0:
return python_tests

for script_info in parsed_scripts["tests"]:
test_function = script_info["function"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,18 @@
# limitations under the License.
#

import asyncio
from pathlib import Path
from typing import Optional

from ..models.sdk_test_folder import SDKTestFolder
from ..paths import SDK_CHECKOUT_PATH, SDK_TESTS_ROOT
from ..paths import SDK_CHECKOUT_PATH
from .list_python_tests_classes import (
CUSTOM_PYTHON_SCRIPTS_FOLDER,
CUSTOM_PYTHON_TESTS_PARSED_FILE,
PYTHON_TESTS_PARSED_FILE,
generate_python_test_json_file,
)
from .models.python_test_models import PythonTest, PythonTestType
from .models.python_test_parser import parse_python_script
from .models.test_declarations import (
Expand All @@ -44,13 +51,6 @@
path=SDK_PYTHON_TEST_PATH, filename_pattern="TC*"
)

CUSTOM_PYTHON_TEST_PATH = PYTHON_TEST_PATH / "custom"
CUSTOM_PYTHON_TEST_FOLDER = SDKTestFolder(
path=CUSTOM_PYTHON_TEST_PATH, filename_pattern="TC*"
)

PYTHON_TESTS_PARSED_FILE = SDK_TESTS_ROOT / "python_tests_info.json"


def _init_test_suites(
python_test_version: str,
Expand Down Expand Up @@ -169,27 +169,32 @@ def sdk_mandatory_python_test_collection(


def custom_python_test_collection(
python_test_folder: SDKTestFolder = CUSTOM_PYTHON_TEST_FOLDER,
python_test_folder: SDKTestFolder = CUSTOM_PYTHON_SCRIPTS_FOLDER,
tests_file_path: Path = CUSTOM_PYTHON_TESTS_PARSED_FILE,
) -> Optional[PythonCollectionDeclaration]:
asyncio.run(
generate_python_test_json_file(
test_folder=python_test_folder,
json_output_file=tests_file_path,
)
)

"""Declare a new collection of test suites."""
return None
# TODO: Implement custom python test collection
# collection = PythonCollectionDeclaration(
# name="Custom SDK Python Tests", folder=python_test_folder
# )

# suites = __parse_python_tests(
# python_test_version="custom",
# mandatory=False,
# )

# for suite in suites:
# if not suite.test_cases:
# continue
# suite.sort_test_cases()
# collection.add_test_suite(suite)

# if not collection.test_suites:
# return None

# return collection
collection = PythonCollectionDeclaration(
name="Custom SDK Python Tests", folder=python_test_folder
)

suites = __parse_python_tests(
python_test_version="custom", mandatory=False, tests_file_path=tests_file_path
)

for suite in suites:
if not suite.test_cases:
continue
suite.sort_test_cases()
collection.add_test_suite(suite)

if not collection.test_suites:
return None

return collection
Loading

0 comments on commit 9d79b81

Please sign in to comment.