Merge branch 'main' into lazy_evaluation_test_step_names

* main:
  Stop cloning data model directory (#112)
  Python test scripts validator (#128)
  v2.11-beta2+fall2024 (#121)
  Address the location change of the otbr scripts (#127)
  Use manual-code or discriminator and password, never all of them (#125)
  Fix python test steps handling (#124)
  PICS v2 support (#122)
  support wifipaf-wifi command for test harness (#117)
  Reverting workaround due to log being truncated as sdk already fix the problem. (#120)
  Remove 22.04 packages versions (#119)

# Conflicts:
#	test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py
#	test_collections/matter/sdk_tests/support/python_testing/models/test_case.py

.version_information

@@ -1 +1 @@
-v2.11-beta1+fall2024
+v2.11-beta2+fall2024

alembic/versions/e2c185af1226_pics_v2_support.py

@@ -0,0 +1,70 @@
+"""pics_v2_support
+
+Revision ID: e2c185af1226
+Revises: 9df8004ad9bb
+Create Date: 2024-06-19 11:46:15.158526
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "e2c185af1226"
+down_revision = "9df8004ad9bb"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "testrunexecution",
+        sa.Column("certification_mode", sa.Boolean(), nullable=True, default=False),
+    )
+    op.add_column(
+        "testsuiteexecution",
+        sa.Column("mandatory", sa.Boolean(), nullable=True, default=False),
+    )
+
+    op.add_column(
+        "testsuitemetadata",
+        sa.Column("mandatory", sa.Boolean(), nullable=True, default=False),
+    )
+    op.add_column(
+        "testcasemetadata",
+        sa.Column("mandatory", sa.Boolean(), nullable=True, default=False),
+    )
+
+    op.execute("UPDATE testrunexecution SET certification_mode = false")
+    op.execute("UPDATE testsuiteexecution SET mandatory = false")
+    op.execute("UPDATE testsuitemetadata SET mandatory = false")
+    op.execute("UPDATE testcasemetadata SET mandatory = false")
+
+    op.alter_column(
+        "testrunexecution",
+        "certification_mode",
+        existing_type=sa.Boolean(),
+        nullable=False,
+    )
+    op.alter_column(
+        "testsuiteexecution", "mandatory", existing_type=sa.Boolean(), nullable=False
+    )
+    op.alter_column(
+        "testsuitemetadata",
+        sa.Column("mandatory", nullable=False, default=False),
+    )
+    op.alter_column(
+        "testcasemetadata",
+        sa.Column("mandatory", nullable=False, default=False),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("testrunexecution", "certification_mode")
+    op.drop_column("testsuiteexecution", "mandatory")
+    op.drop_column("testsuitemetadata", "mandatory")
+    op.drop_column("testcasemetadata", "mandatory")
+    # ### end Alembic commands ###

app/api/api_v1/endpoints/test_run_executions.py

@@ -78,11 +78,13 @@ def create_test_run_execution(
     db: Session = Depends(get_db),
     test_run_execution_in: schemas.TestRunExecutionCreate,
     selected_tests: schemas.TestSelection,
+    certification_mode: bool = False,
 ) -> TestRunExecution:
     """Create a new test run execution."""
 
     # TODO: Remove test_run_config completely from the project
     test_run_execution_in.test_run_config_id = None
+    test_run_execution_in.certification_mode = certification_mode
 
     test_run_execution = crud.test_run_execution.create(
         db=db, obj_in=test_run_execution_in, selected_tests=selected_tests
@@ -260,6 +262,7 @@ def repeat_test_run_execution(
     test_run_execution_in.description = execution_to_repeat.description
     test_run_execution_in.project_id = execution_to_repeat.project_id
     test_run_execution_in.operator_id = execution_to_repeat.operator_id
+    test_run_execution_in.certification_mode = execution_to_repeat.certification_mode
     # TODO: Remove test_run_config completely from the project
     test_run_execution_in.test_run_config_id = None
 

app/crud/crud_test_run_execution.py

@@ -164,6 +164,25 @@ def __load_stats(
 
         return result
 
+    def __sort_selected_tests(
+        self, selected_tests: List[TestSuiteExecution]
+    ) -> List[TestSuiteExecution]:
+        """Sorts the selected tests, make the mandatories test cases the first to be
+        returned."""
+        sorted_selected_tests = []
+
+        # First add the mandatories test cases
+        for suite in selected_tests:
+            if suite.mandatory:
+                sorted_selected_tests.append(suite)
+
+        # Add the remaining test cases
+        for suite in selected_tests:
+            if not suite.mandatory:
+                sorted_selected_tests.append(suite)
+
+        return sorted_selected_tests
+
     def create(
         self,
         db: Session,
@@ -196,7 +215,9 @@ def create(
             )
         )
 
-        test_run_execution.test_suite_executions.extend(test_suites)
+        # Sorting test_suite according to mandatories suites
+        test_suites_sorted = self.__sort_selected_tests(test_suites)
+        test_run_execution.test_suite_executions.extend(test_suites_sorted)
 
         db.commit()
         db.refresh(test_run_execution)

app/models/test_case_metadata.py

@@ -35,6 +35,7 @@ class TestCaseMetadata(Base):
     description: Mapped[str] = mapped_column(Text, nullable=False)
     version: Mapped[str] = mapped_column(nullable=False)
     source_hash: Mapped[str] = mapped_column(VARCHAR(64), nullable=False, index=True)
+    mandatory: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     created_at: Mapped[datetime] = mapped_column(default=datetime.now, nullable=False)
 

app/models/test_enums.py

@@ -24,5 +24,5 @@ class TestStateEnum(str, Enum):
     PASSED = "passed"  # Test Passed with no issued
     FAILED = "failed"  # Test Failed
     ERROR = "error"  # Test Error due to tool setup or environment
-    NOT_APPLICABLE = "not_applicable"  # TODO: Do we need this for full cert runs?
+    NOT_APPLICABLE = "not_applicable"  # Test is not applicable - e.g. PICS mismatch
     CANCELLED = "cancelled"

app/models/test_run_execution.py

@@ -50,6 +50,7 @@ class TestRunExecution(Base):
     completed_at: Mapped[Optional[datetime]]
     archived_at: Mapped[Optional[datetime]]
     imported_at: Mapped[Optional[datetime]]
+    certification_mode: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     description: Mapped[Optional[str]] = mapped_column(default=None, nullable=True)
 

app/models/test_suite_execution.py

@@ -38,6 +38,7 @@ class TestSuiteExecution(Base):
     public_id: Mapped[str] = mapped_column(nullable=False)
     execution_index: Mapped[int] = mapped_column(nullable=False)
     collection_id: Mapped[str] = mapped_column(nullable=False)
+    mandatory: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     state: Mapped[TestStateEnum] = mapped_column(
         Enum(TestStateEnum), nullable=False, default=TestStateEnum.PENDING

app/models/test_suite_metadata.py

@@ -35,6 +35,7 @@ class TestSuiteMetadata(Base):
     description: Mapped[str] = mapped_column(Text, nullable=False)
     version: Mapped[str] = mapped_column(nullable=False)
     source_hash: Mapped[str] = mapped_column(VARCHAR(64), nullable=False, index=True)
+    mandatory: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     created_at: Mapped[datetime] = mapped_column(default=datetime.now)
 

app/pics_applicable_test_cases.py

@@ -13,9 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from typing import Dict
+
 from loguru import logger
 
 from app.schemas.pics import PICS, PICSApplicableTestCases
+from app.test_engine.models.test_declarations import TestCollectionDeclaration
 from app.test_engine.test_script_manager import test_script_manager
 
 
@@ -29,7 +32,7 @@ def applicable_test_cases_list(pics: PICS) -> PICSApplicableTestCases:
         PICSApplicableTestCases: List of test cases that are applicable
           for this Project
     """
-    applicable_tests: set = set()
+    applicable_tests: list = []
 
     if len(pics.clusters) == 0:
         # If the user has not uploaded any PICS
@@ -40,15 +43,37 @@ def applicable_test_cases_list(pics: PICS) -> PICSApplicableTestCases:
     test_collections = test_script_manager.test_collections
     enabled_pics = set([item.number for item in pics.all_enabled_items()])
 
-    for test_collection in test_collections.values():
-        for test_suite in test_collection.test_suites.values():
-            for test_case in test_suite.test_cases.values():
-                if len(test_case.pics) == 0:
-                    # Test cases without pics required are always applicable
-                    applicable_tests.add(test_case.metadata["title"])
-                elif len(test_case.pics) > 0:
-                    if test_case.pics.issubset(enabled_pics):
-                        applicable_tests.add(test_case.metadata["title"])
+    applicable_mandatories_tests = __applicable_test_cases(
+        test_collections, enabled_pics, True
+    )
+    applicable_remaining_tests = __applicable_test_cases(
+        test_collections, enabled_pics, False
+    )
+
+    # Add first the mandatories test cases
+    applicable_tests.extend(applicable_mandatories_tests)
+    # Add the remaining test cases
+    applicable_tests.extend(applicable_remaining_tests)
 
     logger.debug(f"Applicable test cases: {applicable_tests}")
     return PICSApplicableTestCases(test_cases=applicable_tests)
+
+
+def __applicable_test_cases(
+    test_collections: Dict[str, TestCollectionDeclaration],
+    enabled_pics: set[str],
+    mandatory: bool,
+) -> list:
+    applicable_tests: list = []
+
+    for test_collection in test_collections.values():
+        if test_collection.mandatory == mandatory:
+            for test_suite in test_collection.test_suites.values():
+                for test_case in test_suite.test_cases.values():
+                    if len(test_case.pics) == 0:
+                        # Test cases without pics required are always applicable
+                        applicable_tests.append(test_case.metadata["title"])
+                    elif len(test_case.pics) > 0:
+                        if test_case.pics.issubset(enabled_pics):
+                            applicable_tests.append(test_case.metadata["title"])
+    return applicable_tests

app/schemas/pics.py

@@ -38,7 +38,7 @@ def all_enabled_items(self) -> list[PICSItem]:
 
 
 class PICSApplicableTestCases(BaseModel):
-    test_cases: set[str]
+    test_cases: list[str]
 
 
 class PICSError(Exception):

app/schemas/test_case_metadata.py

@@ -26,6 +26,7 @@ class TestCaseMetadataBase(BaseModel):
     description: str
     version: str
     source_hash: str
+    mandatory: bool = False
 
     class Config:
         orm_mode = True

app/schemas/test_collections.py

@@ -23,6 +23,7 @@ class TestMetadata(BaseModel):
     version: str
     title: str
     description: str
+    mandatory: bool = False
 
 
 class TestCase(BaseModel):

app/schemas/test_run_execution.py

@@ -39,6 +39,7 @@ class TestRunExecutionBase(BaseModel):
 
     title: str
     description: Optional[str]
+    certification_mode: bool = False
 
 
 # Base + properties that represent relationhips

app/schemas/test_suite_execution.py

@@ -31,6 +31,7 @@ class TestSuiteExecutionBase(BaseModel):
     public_id: str
     execution_index: int
     collection_id: str
+    mandatory: bool = False
 
 
 # Properties shared by models stored in DB

app/schemas/test_suite_metadata.py

@@ -26,6 +26,7 @@ class TestSuiteMetadataBase(BaseModel):
     description: str
     version: str
     source_hash: str
+    mandatory: bool = False
 
     class Config:
         orm_mode = True

app/test_engine/models/test_case.py

@@ -130,6 +130,11 @@ def __compute_state(self) -> TestStateEnum:
         if self.errors is not None and len(self.errors) > 0:
             return TestStateEnum.ERROR
 
+        # Test cases that have already been marked as not applicable should not
+        # change state
+        if self.state == TestStateEnum.NOT_APPLICABLE:
+            return TestStateEnum.NOT_APPLICABLE
+
         # Note: These loops cannot be easily coalesced as we need to iterate through
         # and assign Test Case State in order.
         if self.any_steps_with_state(TestStateEnum.CANCELLED):
@@ -147,10 +152,14 @@ def __compute_state(self) -> TestStateEnum:
         return TestStateEnum.PASSED
 
     def any_steps_with_state(self, state: TestStateEnum) -> bool:
-        return any(ts for ts in self.test_steps if ts.state == state)
+        return any(ts.state == state for ts in self.test_steps)
 
     def completed(self) -> bool:
-        return self.state not in [TestStateEnum.PENDING, TestStateEnum.EXECUTING]
+        return self.state not in [
+            TestStateEnum.PENDING,
+            TestStateEnum.EXECUTING,
+            TestStateEnum.NOT_APPLICABLE,
+        ]
 
     def __cancel_remaning_test_steps(self) -> None:
         for step in self.test_steps:
@@ -171,7 +180,9 @@ def mark_as_completed(self) -> None:
         if self.completed():
             return
         self.state = self.__compute_state()
-        logger.info(f"Test Case Completed[{self.state.name}]: {self.metadata['title']}")
+        logger.info(
+            f"Test Case Completed [{self.state.name}]: {self.metadata['title']}"
+        )
         self.__print_log_separator()
 
     def mark_as_executing(self) -> None:
@@ -269,6 +280,10 @@ def mark_step_failure(self, msg: Union[str, Exception]) -> None:
 
         self.current_test_step.append_failure(message)
 
+    def mark_as_not_applicable(self) -> None:
+        self.state = TestStateEnum.NOT_APPLICABLE
+        logger.warning(f"Test Case Not Applicable: {self.metadata['public_id']}")
+
     def next_step(self) -> None:
         if self.current_test_step_index + 1 >= len(self.test_steps):
             return

app/test_engine/models/test_declarations.py

@@ -42,9 +42,10 @@ def public_id(self) -> str:
 
 
 class TestSuiteDeclaration(object):
-    def __init__(self, class_ref: Type[TestSuite]) -> None:
+    def __init__(self, class_ref: Type[TestSuite], mandatory: bool = False) -> None:
         self.class_ref = class_ref
         self.test_cases: Dict[str, TestCaseDeclaration] = {}
+        self.mandatory: bool = mandatory
 
     @property
     def public_id(self) -> str:
@@ -100,10 +101,11 @@ def metadata(self) -> TestMetadata:
 
 
 class TestCollectionDeclaration(object):
-    def __init__(self, path: str, name: str) -> None:
+    def __init__(self, path: str, name: str, mandatory: bool = False) -> None:
         self.name = name
         self.path = path
         self.test_suites: Dict[str, TestSuiteDeclaration] = {}
+        self.mandatory = mandatory
 
     def add_test_suite(self, suite: TestSuiteDeclaration) -> None:
         self.test_suites[suite.public_id] = suite