diff --git a/src/python_testing/TC_DeviceConformance.py b/src/python_testing/TC_DeviceConformance.py index 3cc57e2b018ab6..060228a5170f17 100644 --- a/src/python_testing/TC_DeviceConformance.py +++ b/src/python_testing/TC_DeviceConformance.py @@ -50,7 +50,7 @@ async def setup_class_helper(self): self.xml_device_types, problems = build_xml_device_types() self.problems.extend(problems) - def check_conformance(self, ignore_in_progress: bool, is_ci: bool): + def check_conformance(self, ignore_in_progress: bool, is_ci: bool, allow_provisional: bool): problems = [] success = True @@ -87,32 +87,27 @@ def record_warning(location, problem): ignore_attributes.update(ci_ignore_attributes) success = True - allow_provisional = self.user_params.get("allow_provisional", False) - # TODO: automate this once https://github.com/csa-data-model/projects/issues/454 is done. - provisional_cluster_ids = [Clusters.ContentControl.id, Clusters.ScenesManagement.id, Clusters.BallastConfiguration.id, - Clusters.EnergyPreference.id, Clusters.DeviceEnergyManagement.id, Clusters.DeviceEnergyManagementMode.id, Clusters.PulseWidthModulation.id, - Clusters.ProxyConfiguration.id, Clusters.ProxyDiscovery.id, Clusters.ProxyValid.id] - # TODO: Remove this once the latest 1.3 lands with the clusters removed from the DM XML and change the warning below about missing DM XMLs into a proper error - # These are clusters that weren't part of the 1.3 spec that landed in the SDK before the branch cut + provisional_cluster_ids = [] + # TODO: Remove this once we have a scrape without items not going to the test events + # These are clusters that weren't part of the 1.3 or 1.4 spec that landed in the SDK before the branch cut + # They're not marked provisional, but are present in the ToT spec under an ifdef. provisional_cluster_ids.extend([Clusters.DemandResponseLoadControl.id]) - # These clusters are zigbee only. I don't even know why they're part of the codegen, but we should get rid of them. - provisional_cluster_ids.extend([Clusters.BarrierControl.id, Clusters.OnOffSwitchConfiguration.id, - Clusters.BinaryInputBasic.id, Clusters.ElectricalMeasurement.id]) + for endpoint_id, endpoint in self.endpoints_tlv.items(): for cluster_id, cluster in endpoint.items(): cluster_location = ClusterPathLocation(endpoint_id=endpoint_id, cluster_id=cluster_id) - if not allow_provisional and cluster_id in provisional_cluster_ids: - record_error(location=cluster_location, problem='Provisional cluster found on device') - continue - if cluster_id not in self.xml_clusters.keys(): if (cluster_id & 0xFFFF_0000) != 0: # manufacturer cluster continue - # TODO: update this from a warning once we have all the data - record_warning(location=cluster_location, - problem='Standard cluster found on device, but is not present in spec data') + record_error(location=cluster_location, + problem='Standard cluster found on device, but is not present in spec data') + continue + + is_provisional = cluster_id in provisional_cluster_ids or self.xml_clusters[cluster_id].is_provisional + if not allow_provisional and is_provisional: + record_error(location=cluster_location, problem='Provisional cluster found on device') continue feature_map = cluster[GlobalAttributeIds.FEATURE_MAP_ID] @@ -340,7 +335,8 @@ def test_TC_IDM_10_2(self): # https://github.com/project-chip/connectedhomeip/issues/34615 ignore_in_progress = self.user_params.get("ignore_in_progress", True) is_ci = self.check_pics('PICS_SDK_CI_ONLY') - success, problems = self.check_conformance(ignore_in_progress, is_ci) + allow_provisional = self.user_params.get("allow_provisional", False) + success, problems = self.check_conformance(ignore_in_progress, is_ci, allow_provisional) self.problems.extend(problems) if not success: self.fail_current_test("Problems with conformance") diff --git a/src/python_testing/TestConformanceTest.py b/src/python_testing/TestConformanceTest.py new file mode 100644 index 00000000000000..1fc25af9451733 --- /dev/null +++ b/src/python_testing/TestConformanceTest.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import xml.etree.ElementTree as ElementTree +from typing import Any + +import chip.clusters as Clusters +from TC_DeviceConformance import DeviceConformanceTests +from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main +from spec_parsing_support import build_xml_clusters, build_xml_device_types +from mobly import asserts + + +def create_onoff_endpoint(endpoint: int) -> dict[int, dict[int, dict[int, Any]]]: + # Really simple device with one endpoint that includes scenes management, which is provisional + # I'm ONLY populating the global attributes since the conformance test only uses these. + endpoint_tlv = {endpoint: {}} + + on_off_device_type_id = 0x0100 + on_off_device_type_revision = 3 + + # Descriptor + attr = Clusters.Descriptor.Attributes + attrs = {} + attrs[attr.FeatureMap.attribute_id] = 0 + attrs[attr.AcceptedCommandList.attribute_id] = [] + attrs[attr.GeneratedCommandList.attribute_id] = [] + attrs[attr.ClusterRevision.attribute_id] = 3 + attrs[attr.DeviceTypeList.attribute_id] = [Clusters.Descriptor.Structs.DeviceTypeStruct( + deviceType=on_off_device_type_id, revision=on_off_device_type_revision)] + attrs[attr.ServerList.attribute_id] = [Clusters.Identify.id, + Clusters.Groups.id, Clusters.ScenesManagement.id, Clusters.OnOff.id] + attrs[attr.ClientList.attribute_id] = [] + attrs[attr.PartsList.attribute_id] = [] + attrs[attr.AttributeList.attribute_id] = [] + attrs[attr.AttributeList.attribute_id] = list[attrs.keys()] + + endpoint_tlv[endpoint][Clusters.Descriptor.id] = attrs + + # Identify + attr = Clusters.Identify.Attributes + attrs = {} + attrs[attr.FeatureMap.attribute_id] = 0 + attrs[attr.AcceptedCommandList.attribute_id] = [Clusters.Identify.Commands.Identify.command_id] + attrs[attr.GeneratedCommandList.attribute_id] = [] + attrs[attr.ClusterRevision.attribute_id] = 5 + attrs[attr.IdentifyTime.attribute_id] = 0 + attrs[attr.IdentifyType.attribute_id] = Clusters.Identify.Enums.IdentifyTypeEnum.kNone + attrs[attr.AttributeList.attribute_id] = [] + attrs[attr.AttributeList.attribute_id] = list[attrs.keys()] + + endpoint_tlv[endpoint][Clusters.Identify.id] = attrs + + # OnOff + attr = Clusters.OnOff.Attributes + attrs = {} + # device type requires LT feature + attrs[attr.FeatureMap.attribute_id] = Clusters.OnOff.Bitmaps.Feature.kLighting + cmd = Clusters.OnOff.Commands + attrs[attr.AcceptedCommandList.attribute_id] = [cmd.Off.command_id, cmd.On.command_id, cmd.Toggle.command_id, + cmd.OffWithEffect.command_id, cmd.OnWithRecallGlobalScene.command_id, cmd.OnWithTimedOff.command_id] + attrs[attr.GeneratedCommandList.attribute_id] = [] + attrs[attr.ClusterRevision.attribute_id] = 6 + attrs[attr.OnOff.attribute_id] = False + attrs[attr.GlobalSceneControl.attribute_id] = False + attrs[attr.OnTime.attribute_id] = 0 + attrs[attr.OffWaitTime.attribute_id] = 0 + attrs[attr.StartUpOnOff.attribute_id] = Clusters.OnOff.Enums.StartUpOnOffEnum.kOff + attrs[attr.AttributeList.attribute_id] = [] + attrs[attr.AttributeList.attribute_id] = list[attrs.keys()] + + endpoint_tlv[endpoint][Clusters.OnOff.id] = attrs + + # Scenes + attr = Clusters.ScenesManagement.Attributes + attrs = {} + attrs[attr.FeatureMap.attribute_id] = 0 + cmd = Clusters.ScenesManagement.Commands + attrs[attr.AcceptedCommandList.attribute_id] = [cmd.AddScene.command_id, cmd.ViewScene.command_id, cmd.RemoveScene.command_id, + cmd.RemoveAllScenes.command_id, cmd.StoreScene.command_id, cmd.RecallScene.command_id, cmd.GetSceneMembership.command_id] + attrs[attr.GeneratedCommandList.attribute_id] = [cmd.AddSceneResponse.command_id, cmd.ViewSceneResponse.command_id, + cmd.RemoveSceneResponse.command_id, cmd.RemoveAllScenesResponse.command_id, + cmd.StoreSceneResponse.command_id, cmd.GetSceneMembershipResponse.command_id] + attrs[attr.ClusterRevision.attribute_id] = 1 + attrs[attr.SceneTableSize.attribute_id] = 16 + attrs[attr.FabricSceneInfo.attribute_id] = [] + attrs[attr.AttributeList.attribute_id] = [] + attrs[attr.AttributeList.attribute_id] = list[attrs.keys()] + + endpoint_tlv[endpoint][Clusters.ScenesManagement.id] = attrs + + return endpoint_tlv + + +class TestConformanceSupport(MatterBaseTest, DeviceConformanceTests): + def setup_class(self): + self.xml_clusters, self.problems = build_xml_clusters() + self.xml_device_types, problems = build_xml_device_types() + self.problems.extend(problems) + + @async_test_body + async def test_provisional_cluster(self): + # NOTE: I'm actually FORCING scenes to provisional in this test because it will not be provisional + # forever. + self.xml_clusters[Clusters.ScenesManagement.id].is_provisional = True + + self.endpoints_tlv = create_onoff_endpoint(1) + + # The CI flag here is to deal with example code that improperly implements the network commissioning cluster. + # It does not apply here. + success, problems = self.check_conformance(ignore_in_progress=False, is_ci=False, allow_provisional=False) + asserts.assert_false(success, "Unexpected success parsing endpoint with provisional cluster") + + success, problems = self.check_conformance(ignore_in_progress=False, is_ci=False, allow_provisional=True) + asserts.assert_true(success, "Unexpected failure parsing endpoint with provisional cluster and allow_provisional enabled") + + self.xml_clusters[Clusters.ScenesManagement.id].is_provisional = False + success, problems = self.check_conformance(ignore_in_progress=False, is_ci=False, allow_provisional=False) + asserts.assert_true(success, "Unexpected failure parsing endpoint with no clusters marked as provisional") + + +if __name__ == "__main__": + default_matter_test_main() diff --git a/src/python_testing/TestSpecParsingSupport.py b/src/python_testing/TestSpecParsingSupport.py index 0523ebbcd6a6df..efd4e333437d36 100644 --- a/src/python_testing/TestSpecParsingSupport.py +++ b/src/python_testing/TestSpecParsingSupport.py @@ -15,6 +15,7 @@ # limitations under the License. # +import jinja2 import os import xml.etree.ElementTree as ElementTree @@ -225,6 +226,28 @@ def get_access_enum_from_string(access_str: str) -> Clusters.AccessControl.Enums '' ) +PROVISIONAL_CLUSTER_TEMPLATE = """ + + + + + + + {% if provisional %} + + {% endif %} + + + + + + + + + + +""" + class TestSpecParsingSupport(MatterBaseTest): def setup_class(self): @@ -406,6 +429,32 @@ def test_known_derived_clusters(self): for d in known_derived_clusters: asserts.assert_true(self.spec_xml_clusters is not None, "Derived cluster with no base cluster marker") + def test_provisional_clusters(self): + clusters: dict[int, XmlCluster] = {} + pure_base_clusters: dict[str, XmlCluster] = {} + ids_by_name: dict[str, int] = {} + problems: list[ProblemNotice] = [] + id = 0x0001 + + environment = jinja2.Environment() + template = environment.from_string(PROVISIONAL_CLUSTER_TEMPLATE) + + provisional = template.render(provisional=True, id=id) + cluster_xml = ElementTree.fromstring(provisional) + add_cluster_data_from_xml(cluster_xml, clusters, pure_base_clusters, ids_by_name, problems) + + asserts.assert_equal(len(problems), 0, "Unexpected problems parsing provisional cluster") + asserts.assert_in(id, clusters.keys(), "Provisional cluster not parsed") + asserts.assert_true(clusters[id].is_provisional, "Provisional cluster not marked as provisional") + + non_provisional = template.render(provisional=False, id=id) + cluster_xml = ElementTree.fromstring(non_provisional) + add_cluster_data_from_xml(cluster_xml, clusters, pure_base_clusters, ids_by_name, problems) + + asserts.assert_equal(len(problems), 0, "Unexpected problems parsing non-provisional cluster") + asserts.assert_in(id, clusters.keys(), "Non-provisional cluster not parsed") + asserts.assert_false(clusters[id].is_provisional, "Non-provisional cluster marked as provisional") + if __name__ == "__main__": default_matter_test_main() diff --git a/src/python_testing/spec_parsing_support.py b/src/python_testing/spec_parsing_support.py index 1b8e29c4cf33cf..e64a6ebbdcf232 100644 --- a/src/python_testing/spec_parsing_support.py +++ b/src/python_testing/spec_parsing_support.py @@ -108,6 +108,7 @@ class XmlCluster: unknown_commands: list[XmlCommand] events: dict[uint, XmlEvent] pics: str + is_provisional: bool class ClusterSide(Enum): @@ -206,6 +207,7 @@ def __init__(self, cluster, cluster_id, name): self._name = name self._derived = None + self._is_provisional = False try: classification = next(cluster.iter('classification')) hierarchy = classification.attrib['hierarchy'] @@ -214,6 +216,9 @@ def __init__(self, cluster, cluster_id, name): except (KeyError, StopIteration): self._derived = None + if list(cluster.iter('provisionalConform')): + self._is_provisional = True + try: classification = next(cluster.iter('classification')) self._pics = classification.attrib['picsCode'] @@ -451,7 +456,7 @@ def create_cluster(self) -> XmlCluster: accepted_commands=self.parse_commands(CommandType.ACCEPTED), generated_commands=self.parse_commands(CommandType.GENERATED), unknown_commands=self.parse_unknown_commands(), - events=self.parse_events(), pics=self._pics) + events=self.parse_events(), pics=self._pics, is_provisional=self._is_provisional) def get_problems(self) -> list[ProblemNotice]: return self._problems @@ -663,11 +668,13 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt generated_commands[cmd.id].conformance = cmd.conformance else: unknown_commands.append(cmd) + provisional = c.is_provisional or base.is_provisional new = XmlCluster(revision=c.revision, derived=c.derived, name=c.name, feature_map=feature_map, attribute_map=attribute_map, command_map=command_map, features=features, attributes=attributes, accepted_commands=accepted_commands, - generated_commands=generated_commands, unknown_commands=unknown_commands, events=events, pics=c.pics) + generated_commands=generated_commands, unknown_commands=unknown_commands, events=events, pics=c.pics, + is_provisional=provisional) xml_clusters[id] = new