Move SDK informations to Matter program folder (#78)

* Move sdk info to matter program folder

* fix fetch sdk stuff script

* rename matterSettings
fix matter module load in version.py

* Fix sdk fetch scripts

* Fix import references

* Update comments

* Read matter sdk sha only if exists

* Flake8 fixes

* Black fixes

* Isort fixes

* Isort fixes

* Isort fixes

app/core/config.py

@@ -107,16 +107,6 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool:
     NOTIFIER_TO: str = "[email protected]"
     NOTIFIER_SUBJECT: str = "CHIP Tool Crash Log"
 
-    # Test Engine Config
-    CHIP_TOOL_TRACE: bool = True
-    SDK_CONTAINER_NAME: str = "th-sdk"
-
-    # SDK Docker Image
-    SDK_DOCKER_IMAGE: str = "connectedhomeip/chip-cert-bins"
-    SDK_DOCKER_TAG: str = "9f6d627e0262e1d023986291948bb4e845be803e"
-    # SDK SHA: used to fetch test YAML from SDK.
-    SDK_SHA: str = "9f6d627e0262e1d023986291948bb4e845be803e"
-
     class Config:
         case_sensitive = True
 

app/test_engine/test_collection_discovery.py

@@ -158,6 +158,7 @@ def __find_classes_of_type(module_name: str, classtype: Type[T]) -> List[Type[T]
 
             except Exception:
                 logger.error(traceback.format_exc())
+                continue
 
             for _, obj in getmembers(submodule):
                 if isclass(obj) and issubclass(obj, classtype):

app/tests/version/test_read_test_harness_backend_version.py

@@ -20,10 +20,10 @@
 import pytest
 
 from app import utils
-from app.core.config import settings
 from app.version import (
     SHA_FILEPATH,
     VERSION_FILEPATH,
+    read_matter_sdk_sha,
     read_test_harness_backend_version,
 )
 
@@ -56,8 +56,10 @@ def test_read_test_harness_backend_version() -> None:
         backend_version = read_test_harness_backend_version()
         assert backend_version.version == expected_version_value
         assert backend_version.sha == expected_sha_value
-        assert backend_version.sdk_sha == settings.SDK_SHA[:7]
         assert backend_version.db_revision == expected_db_revision
+        matter_sdk_sha = read_matter_sdk_sha()
+        if matter_sdk_sha is not None:
+            assert backend_version.sdk_sha == matter_sdk_sha
 
     mock_utils.assert_called_once()
 
@@ -73,7 +75,9 @@ def test_read_test_harness_backend_version_with_empty_files() -> None:
     backend_version = read_test_harness_backend_version()
     assert backend_version.version == expected_version_value
     assert backend_version.sha == expected_sha_value
-    assert backend_version.sdk_sha == settings.SDK_SHA[:7]
+    matter_sdk_sha = read_matter_sdk_sha()
+    if matter_sdk_sha is not None:
+        assert backend_version.sdk_sha == matter_sdk_sha
 
 
 @pytest.mark.serial
@@ -88,4 +92,6 @@ def test_read_test_harness_backend_version_with_missing_files() -> None:
     backend_version = read_test_harness_backend_version()
     assert backend_version.version == expected_version_value
     assert backend_version.sha == expected_sha_value
-    assert backend_version.sdk_sha == settings.SDK_SHA[:7]
+    matter_sdk_sha = read_matter_sdk_sha()
+    if matter_sdk_sha is not None:
+        assert backend_version.sdk_sha == matter_sdk_sha

app/version.py

@@ -13,16 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import importlib
 from pathlib import Path
+from typing import Optional
 
 from loguru import logger
 
 from app import utils
-from app.core.config import settings
 from app.schemas.test_harness_backend_version import TestHarnessBackendVersion
 
 VERSION_FILENAME = ".version_information"
 SHA_FILENAME = ".sha_information"
+MATTER_CONFIG_MODULE = "test_collections.matter.config"
 
 ROOT_PATH = Path(__file__).parent.parent
 
@@ -37,13 +39,12 @@ def read_test_harness_backend_version() -> TestHarnessBackendVersion:
     version_value = utils.read_information_from_file(VERSION_FILEPATH)
     sha_value = utils.read_information_from_file(SHA_FILEPATH)
     db_revision = utils.get_db_revision()
-
-    # Retrieve short SDK SHA from settings (The information is kept in config.py file)
-    sdk_sha_value = settings.SDK_SHA[:7]
+    sdk_sha_value = read_matter_sdk_sha() or ""
 
     logger.info(f"Test Engine version is {version_value}")
     logger.info(f"Test Engine SHA is {sha_value}")
     logger.info(f"Test Engine SDK SHA is {sdk_sha_value}")
+
     return TestHarnessBackendVersion(
         version=version_value,
         sha=sha_value,
@@ -52,4 +53,15 @@ def read_test_harness_backend_version() -> TestHarnessBackendVersion:
     )
 
 
+def read_matter_sdk_sha() -> Optional[str]:
+    """
+    Retrieve short SDK SHA from settings (The information is kept in config.py file)
+    """
+    if importlib.find_loader(MATTER_CONFIG_MODULE) is None:
+        return None
+
+    matter_config_module = importlib.import_module(MATTER_CONFIG_MODULE)
+    return matter_config_module.matter_settings.SDK_SHA[:7]
+
+
 version_information = read_test_harness_backend_version()

test_collections/matter/config.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2024 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from pydantic import BaseSettings
+
+
+class MatterSettings(BaseSettings):
+    # Test Engine Config
+    CHIP_TOOL_TRACE: bool = True
+    SDK_CONTAINER_NAME: str = "th-sdk"
+
+    # SDK Docker Image
+    SDK_DOCKER_IMAGE: str = "connectedhomeip/chip-cert-bins"
+    SDK_DOCKER_TAG: str = "9f6d627e0262e1d023986291948bb4e845be803e"
+    # SDK SHA: used to fetch test YAML from SDK.
+    SDK_SHA: str = "9f6d627e0262e1d023986291948bb4e845be803e"
+
+    class Config:
+        case_sensitive = True
+
+
+matter_settings = MatterSettings()

test_collections/matter/scripts/update-paa-certs.sh

@@ -0,0 +1,76 @@
+#!/bin/bash -e
+
+ #
+ # Copyright (c) 2023 Project CHIP Authors
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+set -x
+set -e
+
+
+# Usage: ./test_collections/matter/scripts/update-paa-certs.sh
+#
+# Will copy PAA certificates from SDK repository
+#
+# Environment variables can be used to override behaviour 
+# - SDK_PATH (When set, SDK_SHA is ignored)
+# - SDK_SHA 
+
+# Paths
+MATTER_PROGRAM_DIR=$(realpath $(dirname "$0")/..)
+TMP_SDK_FOLDER="sdk-sparse"
+TMP_SDK_PATH="/tmp/$TMP_SDK_FOLDER"
+SDK_CERT_PATH="credentials/development/paa-root-certs"
+SDK_CERT_DEVELOPMENT_PATH="credentials/development"
+CERT_PATH="/var/paa-root-certs"
+DEVELOPMENT_PATH="/var/credentials/development"
+
+# If SDK path is not present, then do local checkout
+if [ -z "$SDK_PATH" ]
+then
+
+    # If SDK SHA is not present, then fetch from Matter program config file
+    if [ -z "$SDK_SHA" ]
+    then
+        cd $MATTER_PROGRAM_DIR
+        SDK_SHA=$(cat $MATTER_PROGRAM_DIR/config.py | grep SDK_SHA | cut -d'"' -f 2 | cut -d"'" -f 2)
+    fi
+
+    # Checkout SDK sparsely 
+    rm -rf $TMP_SDK_PATH
+    cd /tmp
+    git clone --filter=blob:none --no-checkout --depth 1 --sparse https://github.com/project-chip/connectedhomeip.git $TMP_SDK_FOLDER
+    cd $TMP_SDK_FOLDER
+    git sparse-checkout init
+    git sparse-checkout set $SDK_CERT_PATH $SDK_CERT_DEVELOPMENT_PATH
+    git checkout -q $SDK_SHA
+    SDK_PATH="$TMP_SDK_PATH"
+fi
+
+# Create folder if missing (owned by user)
+if [ ! -d "$CERT_PATH" ]
+then
+    sudo mkdir -p $CERT_PATH
+    sudo chown $USER:$USER $CERT_PATH
+fi
+
+# Create folder if missing (owned by user)
+if [ ! -d "$DEVELOPMENT_PATH" ]
+then
+    sudo mkdir -p $DEVELOPMENT_PATH
+    sudo chown $USER:$USER $DEVELOPMENT_PATH
+fi
+
+# Copy certs from SDK
+cp "$SDK_PATH/$SDK_CERT_PATH/"* $CERT_PATH/
+cp -R "$SDK_PATH/$SDK_CERT_DEVELOPMENT_PATH/"** $DEVELOPMENT_PATH/

test_collections/matter/scripts/update-sample-apps.sh

@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+ #
+ # Copyright (c) 2023 Project CHIP Authors
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+MATTER_PROGRAM_DIR=$(realpath $(dirname "$0")/..)
+
+printf "\n\n**********"
+printf "\n*** Update Matter Sample Apps ***\n"
+
+# We are fetching SDK docker image and tag name from backend
+# This is done to minimize the places the SDK version is tracked.
+SDK_DOCKER_IMAGE=$(cat $MATTER_PROGRAM_DIR/config.py | grep SDK_DOCKER_IMAGE | cut -d'"' -f 2 | cut -d"'" -f 2)
+SDK_DOCKER_TAG=$(cat $MATTER_PROGRAM_DIR/config.py | grep SDK_DOCKER_TAG | cut -d'"' -f 2 | cut -d"'" -f 2)
+sudo docker pull $SDK_DOCKER_IMAGE:$SDK_DOCKER_TAG
+
+sudo docker run -t -v ~/apps:/apps $SDK_DOCKER_IMAGE:$SDK_DOCKER_TAG bash -c "rm -v /apps/*; cp -v * /apps/;"
+sudo chown -R `whoami` ~/apps

test_collections/matter/sdk_tests/scripts/fetch_sdk_tests_and_runner.sh

@@ -1,6 +1,5 @@
 #!/bin/bash -e
-set -x
-set -e
+
  #
  # Copyright (c) 2023 Project CHIP Authors
  #
@@ -20,9 +19,11 @@ set -e
 # 
 # When an SDK path is supplied, the SDK_SHA from .env is ignored.
 # Otherwise a temporary checkout of matter sdk will be made.
+set -x
+set -e
 
 # Paths
-ROOT_DIR=$(realpath $(dirname "$0")/../..)
+MATTER_PROGRAM_DIR=$(realpath $(dirname "$0")/../..)
 
 TMP_SDK_FOLDER="sdk-sparse"
 TMP_SDK_PATH="/tmp/$TMP_SDK_FOLDER"
@@ -35,7 +36,7 @@ SDK_EXAMPLE_CHIP_TOOL_PATH="examples/chip-tool"
 SDK_EXAMPLE_PLACEHOLDER_PATH="examples/placeholder"
 SDK_DATA_MODEL_PATH="src/app/zap-templates/zcl/data-model/chip"
 
-TEST_COLLECTIONS_SDK_CHECKOUT_PATH="$ROOT_DIR/sdk_tests/sdk_checkout"
+TEST_COLLECTIONS_SDK_CHECKOUT_PATH="$MATTER_PROGRAM_DIR/sdk_tests/sdk_checkout"
 
 # YAML Files
 YAML_TEST_COLLECTION_PATH="$TEST_COLLECTIONS_SDK_CHECKOUT_PATH/yaml_tests"
@@ -75,8 +76,8 @@ then
     echo "Using custom SDK path: ${SDK_PATH}. Update required"
     SDK_CHECKOUT_VERSION="custom-sdk"
 else
-    # Get configured SDK_SHA (will default to value in app/core/config.py)
-    SDK_SHA=`LOGGING_LEVEL=critical python3 -c "from app.core.config import settings; print(settings.SDK_SHA)"`
+    # Get configured SDK_SHA (will default to value in test_collection/matter/config.py)
+    SDK_SHA=$(cat $MATTER_PROGRAM_DIR/config.py | grep SDK_SHA | cut -d'"' -f 2 | cut -d"'" -f 2)
     if [[ $FORCE_UPDATE -eq 1 ]]
     then 
         echo "Update is forced."

test_collections/matter/sdk_tests/support/chip/chip_server.py

@@ -23,10 +23,10 @@
 
 import loguru
 
-from app.core.config import settings
 from app.singleton import Singleton
 from app.test_engine.logger import CHIPTOOL_LEVEL
 from app.test_engine.logger import test_engine_logger as logger
+from test_collections.matter.config import matter_settings
 
 from ..sdk_container import DOCKER_LOGS_PATH, DOCKER_PAA_CERTS_PATH, SDKContainer
 
@@ -128,7 +128,7 @@ async def start(
         else:
             raise UnsupportedChipServerType(f"Unsupported server type: {server_type}")
 
-        if settings.CHIP_TOOL_TRACE:
+        if matter_settings.CHIP_TOOL_TRACE:
             topic = "CHIP_WEBSOCKET_SERVER"
             command.append(self.trace_file_params(topic))
 

test_collections/matter/sdk_tests/support/sdk_container.py

@@ -22,10 +22,10 @@
 from docker.models.containers import Container
 
 from app.container_manager import container_manager
-from app.core.config import settings
 from app.schemas.pics import PICS, PICSError
 from app.singleton import Singleton
 from app.test_engine.logger import test_engine_logger as logger
+from test_collections.matter.config import matter_settings
 
 from .exec_run_in_container import ExecResultExtended, exec_run_in_container
 from .pics import set_pics_command
@@ -79,8 +79,8 @@ class SDKContainer(metaclass=Singleton):
     Create an instance by calling initializer. When ready to use, ...
     """
 
-    container_name = settings.SDK_CONTAINER_NAME
-    image_tag = f"{settings.SDK_DOCKER_IMAGE}:{settings.SDK_DOCKER_TAG}"
+    container_name = matter_settings.SDK_CONTAINER_NAME
+    image_tag = f"{matter_settings.SDK_DOCKER_IMAGE}:{matter_settings.SDK_DOCKER_TAG}"
     run_parameters = {
         "privileged": True,
         "detach": True,