From 92a8243dbe667b20f8d05a6f7782b8e967062353 Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Fri, 8 Nov 2024 20:48:57 +0000 Subject: [PATCH 01/13] Add zap_cluster_rev tool --- src/tools/zap_cluster_rev/README.md | 74 +++++++++++++++ src/tools/zap_cluster_rev/zap_cluster_rev.py | 97 ++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 src/tools/zap_cluster_rev/README.md create mode 100644 src/tools/zap_cluster_rev/zap_cluster_rev.py diff --git a/src/tools/zap_cluster_rev/README.md b/src/tools/zap_cluster_rev/README.md new file mode 100644 index 00000000000000..579bcb47eb6878 --- /dev/null +++ b/src/tools/zap_cluster_rev/README.md @@ -0,0 +1,74 @@ +--- +orphan: true +--- + +# ZAP Cluster Revision Bump Tool + +This tool parses ZAP files and updates outdated cluster revisions according to +the specification. + +# Prerequisites + +This tool uses the python environment, which can be built and activated using: + +``` +# Build +./scripts/build_python.sh -i out/python_env + +# Activate +source out/python_env/bin/activate +``` + +# How to run + +Usage: + +``` +usage: zap_cluster_rev.py [-h] (--print-only | --update) filenames [filenames ...] +``` + +Example #1: Check cluster revisions and update a ZAP file: + +``` +python src/tools/zap_cluster_rev/zap_cluster_rev.py --update examples/network-manager-app/network-manager-common/network-manager-app.zap +``` + +Expected output if outdated clusters are found: + +``` +python src/tools/zap_cluster_rev/zap_cluster_rev.py --update examples/network-manager-app/network-manager-common/network-manager-app.zap +Checking for outdated cluster revisions on: examples/network-manager-app/network-manager-common/network-manager-app.zap +3 found! +Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning +Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics +Cluster revisions updated successfully! +``` + +Expected output if no outdated clusters are found: + +``` +Checking for outdated cluster revisions on: examples/network-manager-app/network-manager-common/network-manager-app.zap +0 found! +``` + +Example #2: Check the cluster revisions and print only (do not modify the ZAP +file): + +``` +python src/tools/zap_cluster_rev/zap_cluster_rev.py --print-only examples/network-manager-app/network-manager-common/network-manage +r-app.zap +``` + +Expected output: + +``` +Checking for outdated cluster revisions on: examples/network-manager-app/network-manager-common/network-manager-app.zap +3 found! +Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning +Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics +``` + +The option `--print-only` is useful for testing and ensuring the tool is +identifying the right outdated clusters before updating. diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py new file mode 100644 index 00000000000000..703b0330c99553 --- /dev/null +++ b/src/tools/zap_cluster_rev/zap_cluster_rev.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +from chip.testing.spec_parsing import CommandType, build_xml_clusters, build_xml_device_types +from dataclasses import dataclass +import sys +import json +import argparse + + +@dataclass +class ClusterInfo(): + endpoint_id: int + cluster_code: int + cluster_spec_revision: int + cluster_name: str + json_attribute: object + + def cluster_revision(self): + return int(self.json_attribute["defaultValue"]) + + def __str__(self): + return ('Endpoint: %d cluster_code: %d cluster_revision: %d cluster_spec_revision: %d name: %s' % (self.endpoint_id, self.cluster_code, self.cluster_revision(), self.cluster_spec_revision, self.cluster_name)) + + def update_cluster_revision(self): + self.json_attribute["defaultValue"] = self.cluster_spec_revision + + +def load_zap(filename: str): + with open(filename, "rt") as infile: + return json.load(infile) + + +def save_zap(body: object, filename: str): + with open(filename, "wt+") as outfile: + return json.dump(body, outfile, indent=2) + + +def get_outdated_clusters(body: object, xml_clusters: dict) -> list[ClusterInfo]: + result = [] + for endpoint in body.get("endpointTypes", []): + endpoint_id = endpoint.get("id") - 1 + for cluster in endpoint.get("clusters", []): + for attribute in cluster.get("attributes", []): + if attribute.get("name") != "ClusterRevision" or attribute.get("storageOption") != "RAM": + continue + cluster_revision = int(attribute.get("defaultValue")) + spec_revision = xml_clusters[cluster.get("code")].revision + # Filter in outdated clusters only + if (cluster_revision == spec_revision): + break + cluster_info = ClusterInfo(endpoint_id=endpoint_id, cluster_code=cluster.get("code"), + cluster_spec_revision=spec_revision, cluster_name=cluster.get("name"), json_attribute=attribute) + result.append(cluster_info) + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Process ZAP Files and update outdated cluster revisions according to the spec") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--print-only", action="store_true", help="Print outdated cluster information.") + group.add_argument("--update", action="store_true", help="Update outdated cluster revisions on ZAP file.") + parser.add_argument("filenames", nargs="+", help="A sequence of ZAP filenames.") + args = parser.parse_args() + + outdated_count = 0 + for zap_filename in args.filenames: + print("Checking for outdated cluster revisions on: %s" % zap_filename) + body = load_zap(zap_filename) + spec_xml_clusters, problems = build_xml_clusters() + + outdated_clusters = get_outdated_clusters(body, spec_xml_clusters) + print("%d found!" % len(outdated_clusters)) + outdated_count += len(outdated_clusters) + print(*outdated_clusters, sep='\n') + + if (args.print_only): + continue + + # Update outdated cluster revisions according to the spec + for cluster in outdated_clusters: + cluster.update_cluster_revision() + if (outdated_clusters): + print('Cluster revisions updated successfully!\n') + + # Check there's no longer any outdated cluster + assert (not get_outdated_clusters(body, spec_xml_clusters)) + + save_zap(body, zap_filename) + + # If it's printing only, return the number of outdated clusters, so it can be used as a test + if (args.print_only): + return outdated_count + + +if __name__ == "__main__": + main() From 206141cc952d71a2e21c038c76c78cf539ec8dd1 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 11 Nov 2024 15:05:19 +0000 Subject: [PATCH 02/13] Restyled by isort --- src/tools/zap_cluster_rev/zap_cluster_rev.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py index 703b0330c99553..2a5a02f8529fbd 100644 --- a/src/tools/zap_cluster_rev/zap_cluster_rev.py +++ b/src/tools/zap_cluster_rev/zap_cluster_rev.py @@ -1,10 +1,11 @@ #!/usr/bin/env python -from chip.testing.spec_parsing import CommandType, build_xml_clusters, build_xml_device_types -from dataclasses import dataclass -import sys -import json import argparse +import json +import sys +from dataclasses import dataclass + +from chip.testing.spec_parsing import CommandType, build_xml_clusters, build_xml_device_types @dataclass From 53d4c1870e17fdc6b7ef0ec27408bd0e3caaa874 Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Mon, 11 Nov 2024 15:13:01 +0000 Subject: [PATCH 03/13] removed unused import --- src/tools/zap_cluster_rev/zap_cluster_rev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py index 2a5a02f8529fbd..2b58620e4fb61b 100644 --- a/src/tools/zap_cluster_rev/zap_cluster_rev.py +++ b/src/tools/zap_cluster_rev/zap_cluster_rev.py @@ -5,7 +5,7 @@ import sys from dataclasses import dataclass -from chip.testing.spec_parsing import CommandType, build_xml_clusters, build_xml_device_types +from chip.testing.spec_parsing import build_xml_clusters @dataclass From 7dabd92210ddc4cc0b03fd47a908b8d8eefec72c Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Mon, 11 Nov 2024 15:37:41 +0000 Subject: [PATCH 04/13] Add warning note. --- src/tools/zap_cluster_rev/README.md | 3 +++ src/tools/zap_cluster_rev/zap_cluster_rev.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/tools/zap_cluster_rev/README.md b/src/tools/zap_cluster_rev/README.md index 579bcb47eb6878..01a396adb5d8ea 100644 --- a/src/tools/zap_cluster_rev/README.md +++ b/src/tools/zap_cluster_rev/README.md @@ -7,6 +7,9 @@ orphan: true This tool parses ZAP files and updates outdated cluster revisions according to the specification. +**WARNING**: This tool only updates the revision number. Please ensure any new +attributes, events or commands are implemented accordingly. + # Prerequisites This tool uses the python environment, which can be built and activated using: diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py index 2b58620e4fb61b..92e215d158b4ba 100644 --- a/src/tools/zap_cluster_rev/zap_cluster_rev.py +++ b/src/tools/zap_cluster_rev/zap_cluster_rev.py @@ -64,6 +64,7 @@ def main(): parser.add_argument("filenames", nargs="+", help="A sequence of ZAP filenames.") args = parser.parse_args() + print("**WARNING**: This tool only updates the revision number. Please ensure any new attributes, events or commands are implemented accordingly.") outdated_count = 0 for zap_filename in args.filenames: print("Checking for outdated cluster revisions on: %s" % zap_filename) From ab056daee1b74d65c8e0578bc92dc8b03794d35d Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Mon, 11 Nov 2024 15:43:41 +0000 Subject: [PATCH 05/13] Removed unused import --- src/tools/zap_cluster_rev/zap_cluster_rev.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py index 92e215d158b4ba..3e7a07d245c144 100644 --- a/src/tools/zap_cluster_rev/zap_cluster_rev.py +++ b/src/tools/zap_cluster_rev/zap_cluster_rev.py @@ -2,7 +2,6 @@ import argparse import json -import sys from dataclasses import dataclass from chip.testing.spec_parsing import build_xml_clusters From eab3f2b8bb2c4a05aaac4f54aa712f0cd7523d7e Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Mon, 18 Nov 2024 10:08:04 -0500 Subject: [PATCH 06/13] Apply suggestions from code review Co-authored-by: Andrei Litvin --- src/tools/zap_cluster_rev/zap_cluster_rev.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py index 3e7a07d245c144..c2f128c50bd802 100644 --- a/src/tools/zap_cluster_rev/zap_cluster_rev.py +++ b/src/tools/zap_cluster_rev/zap_cluster_rev.py @@ -75,7 +75,7 @@ def main(): outdated_count += len(outdated_clusters) print(*outdated_clusters, sep='\n') - if (args.print_only): + if args.print_only: continue # Update outdated cluster revisions according to the spec @@ -90,7 +90,7 @@ def main(): save_zap(body, zap_filename) # If it's printing only, return the number of outdated clusters, so it can be used as a test - if (args.print_only): + if args.print_only: return outdated_count From ed12b36af8081ac051b972a766b87026c3e2e61c Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Mon, 25 Nov 2024 18:11:48 +0000 Subject: [PATCH 07/13] Move zap_cluster_rev.py into update_cluster_revisions.py --- scripts/tools/zap/README.md | 184 ++++++++++++++++++ scripts/tools/zap/update_cluster_revisions.py | 128 +++++++++--- src/tools/zap_cluster_rev/README.md | 77 -------- src/tools/zap_cluster_rev/zap_cluster_rev.py | 98 ---------- 4 files changed, 287 insertions(+), 200 deletions(-) create mode 100644 scripts/tools/zap/README.md delete mode 100644 src/tools/zap_cluster_rev/README.md delete mode 100644 src/tools/zap_cluster_rev/zap_cluster_rev.py diff --git a/scripts/tools/zap/README.md b/scripts/tools/zap/README.md new file mode 100644 index 00000000000000..8f29fa7cd37902 --- /dev/null +++ b/scripts/tools/zap/README.md @@ -0,0 +1,184 @@ +--- +orphan: true +--- + +# ZAP + +This directory contains various tools related to ZAP. + + +# ZAP Cluster Revision Update Tool (update_cluster_revisions.py) + +This tool parses all ZAP files in the codebase and updates cluster revisions. + +**WARNING**: This tool only updates the revision number. Please ensure any new +attributes, events or commands are implemented accordingly. + +## Prerequisites + +This tool uses the python environment, which can be built and activated using: + +``` +# Build +./scripts/build_python.sh -i out/python_env + +# Activate +source out/python_env/bin/activate +``` + +## How to run + +Usage: + +``` +usage: update_cluster_revisions.py [-h] [--cluster-id CLUSTER_ID] [--new-revision NEW_REVISION] [--old-revision OLD_REVISION] [--dry-run] [--parallel] [--no-parallel] + +Update the ClusterRevision for a chosen cluster in all .zap files + +options: + -h, --help show this help message and exit + --cluster-id CLUSTER_ID + The id of the cluster, as hex, for which the cluster revision should be updated. If omitted, all outdated clusters are updated. + --new-revision NEW_REVISION + The new cluster revision as a decimal integer. If omitted, the cluster revision will be updated to the latest according to the specification + --old-revision OLD_REVISION + If set, only clusters with this old revision will be updated. This is a decimal integer. + --dry-run Don't do any generation, just log what .zap files would be updated (default: False) + --parallel + --no-parallel +``` + +[Note] +* Use `--dry-run` to print only, don't update ZAP files +* Omit `--cluster-id` to search for all clusters +* Omit `--new-revision` to update to the latest revision according to the specification +* Optionally provide `--old-revision`, `--cluster-id` and `--new-revision` to update only clusters that match `old-revision`. + +Example #1: Check all outdated cluster revisions and print only (do not modify the ZAP file `--dry-run`): + +``` +./scripts/tools/zap/update_cluster_revisions.py --dry-run --no-parallel +``` + +Example output if outdated clusters are found: + +``` +... +Checking for outdated cluster revisions on: examples/light-switch-app/light-switch-common/icd-lit-light-switch-app.zap +6 found! +Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning +Endpoint: 0 cluster_code: 49 cluster_revision: 1 cluster_spec_revision: 2 name: Network Commissioning +Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics +Endpoint: 1 cluster_code: 3 cluster_revision: 4 cluster_spec_revision: 5 name: Identify +Endpoint: 2 cluster_code: 3 cluster_revision: 2 cluster_spec_revision: 5 name: Identify +Checking for outdated cluster revisions on: examples/light-switch-app/light-switch-common/light-switch-app.zap +6 found! +Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning +Endpoint: 0 cluster_code: 49 cluster_revision: 1 cluster_spec_revision: 2 name: Network Commissioning +Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics +Endpoint: 1 cluster_code: 3 cluster_revision: 4 cluster_spec_revision: 5 name: Identify +Endpoint: 2 cluster_code: 3 cluster_revision: 2 cluster_spec_revision: 5 name: Identify +Checking for outdated cluster revisions on: examples/light-switch-app/qpg/zap/switch.zap +7 found! +Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Endpoint: 0 cluster_code: 47 cluster_revision: 1 cluster_spec_revision: 3 name: Power Source +Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning +Endpoint: 0 cluster_code: 49 cluster_revision: 1 cluster_spec_revision: 2 name: Network Commissioning +Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics +Endpoint: 1 cluster_code: 3 cluster_revision: 4 cluster_spec_revision: 5 name: Identify +Endpoint: 2 cluster_code: 3 cluster_revision: 4 cluster_spec_revision: 5 name: Identify +Checking for outdated cluster revisions on: src/controller/data_model/controller-clusters.zap +0 found! + +Checking for outdated cluster revisions on: scripts/tools/zap/tests/inputs/lighting-app.zap +8 found! +Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning +Endpoint: 0 cluster_code: 49 cluster_revision: 1 cluster_spec_revision: 2 name: Network Commissioning +Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics +Endpoint: 0 cluster_code: 59 cluster_revision: 1 cluster_spec_revision: 2 name: Switch +Endpoint: 1 cluster_code: 3 cluster_revision: 4 cluster_spec_revision: 5 name: Identify +Endpoint: 1 cluster_code: 6 cluster_revision: 5 cluster_spec_revision: 6 name: On/Off +Endpoint: 1 cluster_code: 1030 cluster_revision: 4 cluster_spec_revision: 5 name: Occupancy Sensing +... +``` + +Example #2: Check for possible outdated revisions of the cluster 0x28 revisions and +print only (do not modify the ZAP file): + +``` +./scripts/tools/zap/update_cluster_revisions.py --dry-run --no-parallel --cluster-id 0x28 +``` + +Example output: + +``` +... +Checking for outdated cluster revisions on: examples/bridge-app/bridge-common/bridge-app.zap +1 found! +Endpoint: 0 cluster_code: 0x28 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Checking for outdated cluster revisions on: examples/window-app/common/window-app.zap +1 found! +Endpoint: 0 cluster_code: 0x28 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Checking for outdated cluster revisions on: examples/all-clusters-minimal-app/all-clusters-common/all-clusters-minimal-app.zap +1 found! +Endpoint: 0 cluster_code: 0x28 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Checking for outdated cluster revisions on: examples/smoke-co-alarm-app/smoke-co-alarm-common/smoke-co-alarm-app.zap +1 found! +Endpoint: 0 cluster_code: 0x28 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Checking for outdated cluster revisions on: examples/placeholder/linux/apps/app1/config.zap +2 found! +Endpoint: 0 cluster_code: 0x28 cluster_revision: 2 cluster_spec_revision: 4 name: Basic Information +Endpoint: 1 cluster_code: 0x28 cluster_revision: 2 cluster_spec_revision: 4 name: Basic Information +... +``` + +Example #3: Update outdated revisions of the cluster 0x28 to the latest in the specification: +``` +./scripts/tools/zap/update_cluster_revisions.py --no-parallel --cluster-id 0x28 +``` +Example output: + +``` +Checking for outdated cluster revisions on: examples/window-app/common/window-app.zap +1 found! +Endpoint: 0 cluster_code: 0x28 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information +Cluster revisions updated successfully! + +Searching for zcl file from /usr/local/google/home/sergiosoares/connectedhomeip/examples/window-app/common/window-app.zap +🔧 Using state directory: /usr/local/google/home/sergiosoares/.zap +🤖 Conversion started + 🔍 input files: /usr/local/google/home/sergiosoares/connectedhomeip/examples/window-app/common/window-app.zap + 🔍 output pattern: /usr/local/google/home/sergiosoares/connectedhomeip/examples/window-app/common/window-app.zap + 🐝 database and schema initialized + 🐝 zcl package loaded: /usr/local/google/home/sergiosoares/connectedhomeip/src/app/zap-templates/zcl/zcl.json + 🐝 templates loaded: /usr/local/google/home/sergiosoares/connectedhomeip/src/app/zap-templates/app-templates.json + + +🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 + +Application is failing the Device Type Specification as follows: + + - ⚠ Check Device Type Compliance on endpoint: 0, device type: MA-rootdevice, cluster: Localization Configuration, attribute: ActiveLocale needs to be enabled + - ⚠ Check Device Type Compliance on endpoint: 0, device type: MA-rootdevice, cluster: Localization Configuration, attribute: SupportedLocales needs to be enabled + - ⚠ Check Device Type Compliance on endpoint: 2, device type: MA-windowcovering, cluster: Window Covering server needs bit 3 enabled in the Feature Map attribute + +Application is failing the Cluster Specification as follows: + + +🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 + + + 👈 read in: /usr/local/google/home/sergiosoares/connectedhomeip/examples/window-app/common/window-app.zap + 👉 write out: /usr/local/google/home/sergiosoares/connectedhomeip/examples/window-app/common/window-app.zap + 👉 write out: undefined +😎 Conversion done! +``` + +Example #4: Update outdated revisions of the cluster 0x28 to version 3: + +``` +./scripts/tools/zap/update_cluster_revisions.py --no-parallel --cluster-id 0x28 --new-revision 3 +``` diff --git a/scripts/tools/zap/update_cluster_revisions.py b/scripts/tools/zap/update_cluster_revisions.py index 797fd1733f3771..5fe96e2e9846e2 100755 --- a/scripts/tools/zap/update_cluster_revisions.py +++ b/scripts/tools/zap/update_cluster_revisions.py @@ -23,6 +23,8 @@ import subprocess import sys from pathlib import Path +from dataclasses import dataclass +from chip.testing.spec_parsing import build_xml_clusters BASIC_INFORMATION_CLUSTER_ID = int("0x0039", 16) CHIP_ROOT_DIR = os.path.realpath( @@ -32,6 +34,24 @@ } +@dataclass +class ClusterInfo(): + endpoint_id: int + cluster_code: int + cluster_spec_revision: int + cluster_name: str + json_attribute: object + + def cluster_revision(self): + return int(self.json_attribute["defaultValue"]) + + def __str__(self): + return ('Endpoint: %d cluster_code: %s cluster_revision: %d cluster_spec_revision: %d name: %s' % (self.endpoint_id, hex(self.cluster_code), self.cluster_revision(), self.cluster_spec_revision, self.cluster_name)) + + def update_cluster_revision(self): + self.json_attribute["defaultValue"] = self.cluster_spec_revision + + def getTargets(cluster_id: int): ROOTS_TO_SEARCH = [ './examples', @@ -51,6 +71,34 @@ def getTargets(cluster_id: int): return targets +def get_outdated_clusters(data: object, xml_clusters: dict, args) -> list[ClusterInfo]: + result = [] + for endpoint in data.get("endpointTypes", []): + endpoint_id = endpoint.get("id") - 1 + for cluster in endpoint.get("clusters", []): + # Skip cluster if user passed a cluster id and it doesn't match + if args.cluster_id is not None and cluster['code'] != args.cluster_id: + continue + for attribute in cluster.get("attributes", []): + if attribute.get("name") != "ClusterRevision" or attribute.get("storageOption") != "RAM": + continue + try: + cluster_revision = int(attribute.get("defaultValue")) + spec_revision = xml_clusters[cluster.get("code")].revision + except: + continue + # Filter in outdated clusters only + if (cluster_revision == spec_revision): + break + # If old_revision is present, filter in matching only + if (args.old_revision is not None and cluster_revision != args.old_revision): + break + cluster_info = ClusterInfo(endpoint_id=endpoint_id, cluster_code=cluster.get("code"), + cluster_spec_revision=spec_revision, cluster_name=cluster.get("name"), json_attribute=attribute) + result.append(cluster_info) + return result + + def checkPythonVersion(): if sys.version_info[0] < 3: print('Must use Python 3. Current version is ' + @@ -61,12 +109,12 @@ def checkPythonVersion(): def runArgumentsParser(): parser = argparse.ArgumentParser( description='Update the ClusterRevision for a chosen cluster in all .zap files') - parser.add_argument('--cluster-id', default=None, action='store', - help='The id of the cluster, as hex, for which the cluster revision should be updated.') - parser.add_argument('--new-revision', default=None, action='store', - help='The new cluster revision as a decimal integer') + parser.add_argument('--cluster-id', required='--new-revision' in sys.argv, default=None, action='store', + help='The id of the cluster, as hex, for which the cluster revision should be updated. If omitted, all outdated clusters are updated.') + parser.add_argument('--new-revision', required='--old-revision' in sys.argv, default=None, action='store', + help='The new cluster revision as a decimal integer. If omitted, the cluster revision will be updated to the latest according to the specification') parser.add_argument('--old-revision', default=None, action='store', - help='If set, only clusters with this old revision will be updated. This is a decimal integer.') + help='If set, only clusters with this old revision will be updated. This is a decimal integer.') parser.add_argument('--dry-run', default=False, action='store_true', help="Don't do any generation, just log what .zap files would be updated (default: False)") parser.add_argument('--parallel', action='store_true') @@ -75,15 +123,8 @@ def runArgumentsParser(): args = parser.parse_args() - if args.cluster_id is None: - logging.error("Must have a cluster id") - sys.exit(1) - - if args.new_revision is None: - logging.error("Must have a new cluster revision") - sys.exit(1) - - args.cluster_id = int(args.cluster_id, 16) + if args.cluster_id: + args.cluster_id = int(args.cluster_id, 16) return args @@ -106,7 +147,11 @@ def updateOne(item): """ Helper method that may be run in parallel to update a single target. """ - (args, target) = item + (args, target, spec_xml_clusters) = item + + print(f"Will try to update: {target}") + if args.dry_run: + return with open(target, "r") as file: data = json.load(file) @@ -126,6 +171,41 @@ def updateOne(item): subprocess.check_call(['./scripts/tools/zap/convert.py', target]) +def updateOneToLatest(item): + """ + Helper method that may be run in parallel to update all clusters in a single .zap file to the latest revision according to the spec. + """ + (args, target, spec_xml_clusters) = item + + with open(target, "r") as file: + data = json.load(file) + + print("Checking for outdated cluster revisions on: %s" % target) + + outdated_clusters = get_outdated_clusters(data, spec_xml_clusters, args) + print("%d found!" % len(outdated_clusters)) + print(*outdated_clusters, sep='\n') + + if args.dry_run: + return + + # Update outdated cluster revisions according to the spec + for cluster in outdated_clusters: + cluster.update_cluster_revision() + + # Check there's no longer any outdated cluster + assert (not get_outdated_clusters(data, spec_xml_clusters, args)) + + # If found outdated clusters, then save and reformat zap + if (outdated_clusters): + print('Cluster revisions updated successfully!\n') + with open(target, "w") as file: + json.dump(data, file) + + # Now run convert.py on the file to have ZAP reformat it however it likes. + subprocess.check_call(['./scripts/tools/zap/convert.py', target]) + + def main(): checkPythonVersion() @@ -136,26 +216,24 @@ def main(): args = runArgumentsParser() - os.chdir(CHIP_ROOT_DIR) + print("**WARNING**: This tool only updates the revision number. Please ensure any new attributes, events or commands are implemented accordingly.") - targets = getTargets(args.cluster_id) - - if args.dry_run: - for target in targets: - print(f"Will try to update: {target}") - sys.exit(0) + os.chdir(CHIP_ROOT_DIR) - items = [(args, target) for target in targets] + targets = getTargets(args.cluster_id if args.cluster_id else 0) + spec_xml_clusters, problems = build_xml_clusters() + items = [(args, target, spec_xml_clusters) for target in targets] + update_func = updateOne if args.new_revision else updateOneToLatest if args.parallel: # Ensure each zap run is independent os.environ['ZAP_TEMPSTATE'] = '1' with multiprocessing.Pool() as pool: - for _ in pool.imap_unordered(updateOne, items): + for _ in pool.imap_unordered(update_func, items): pass else: for item in items: - updateOne(item) + update_func(item) if __name__ == '__main__': diff --git a/src/tools/zap_cluster_rev/README.md b/src/tools/zap_cluster_rev/README.md deleted file mode 100644 index 01a396adb5d8ea..00000000000000 --- a/src/tools/zap_cluster_rev/README.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -orphan: true ---- - -# ZAP Cluster Revision Bump Tool - -This tool parses ZAP files and updates outdated cluster revisions according to -the specification. - -**WARNING**: This tool only updates the revision number. Please ensure any new -attributes, events or commands are implemented accordingly. - -# Prerequisites - -This tool uses the python environment, which can be built and activated using: - -``` -# Build -./scripts/build_python.sh -i out/python_env - -# Activate -source out/python_env/bin/activate -``` - -# How to run - -Usage: - -``` -usage: zap_cluster_rev.py [-h] (--print-only | --update) filenames [filenames ...] -``` - -Example #1: Check cluster revisions and update a ZAP file: - -``` -python src/tools/zap_cluster_rev/zap_cluster_rev.py --update examples/network-manager-app/network-manager-common/network-manager-app.zap -``` - -Expected output if outdated clusters are found: - -``` -python src/tools/zap_cluster_rev/zap_cluster_rev.py --update examples/network-manager-app/network-manager-common/network-manager-app.zap -Checking for outdated cluster revisions on: examples/network-manager-app/network-manager-common/network-manager-app.zap -3 found! -Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information -Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning -Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics -Cluster revisions updated successfully! -``` - -Expected output if no outdated clusters are found: - -``` -Checking for outdated cluster revisions on: examples/network-manager-app/network-manager-common/network-manager-app.zap -0 found! -``` - -Example #2: Check the cluster revisions and print only (do not modify the ZAP -file): - -``` -python src/tools/zap_cluster_rev/zap_cluster_rev.py --print-only examples/network-manager-app/network-manager-common/network-manage -r-app.zap -``` - -Expected output: - -``` -Checking for outdated cluster revisions on: examples/network-manager-app/network-manager-common/network-manager-app.zap -3 found! -Endpoint: 0 cluster_code: 40 cluster_revision: 3 cluster_spec_revision: 4 name: Basic Information -Endpoint: 0 cluster_code: 48 cluster_revision: 1 cluster_spec_revision: 2 name: General Commissioning -Endpoint: 0 cluster_code: 53 cluster_revision: 2 cluster_spec_revision: 3 name: Thread Network Diagnostics -``` - -The option `--print-only` is useful for testing and ensuring the tool is -identifying the right outdated clusters before updating. diff --git a/src/tools/zap_cluster_rev/zap_cluster_rev.py b/src/tools/zap_cluster_rev/zap_cluster_rev.py deleted file mode 100644 index c2f128c50bd802..00000000000000 --- a/src/tools/zap_cluster_rev/zap_cluster_rev.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -import argparse -import json -from dataclasses import dataclass - -from chip.testing.spec_parsing import build_xml_clusters - - -@dataclass -class ClusterInfo(): - endpoint_id: int - cluster_code: int - cluster_spec_revision: int - cluster_name: str - json_attribute: object - - def cluster_revision(self): - return int(self.json_attribute["defaultValue"]) - - def __str__(self): - return ('Endpoint: %d cluster_code: %d cluster_revision: %d cluster_spec_revision: %d name: %s' % (self.endpoint_id, self.cluster_code, self.cluster_revision(), self.cluster_spec_revision, self.cluster_name)) - - def update_cluster_revision(self): - self.json_attribute["defaultValue"] = self.cluster_spec_revision - - -def load_zap(filename: str): - with open(filename, "rt") as infile: - return json.load(infile) - - -def save_zap(body: object, filename: str): - with open(filename, "wt+") as outfile: - return json.dump(body, outfile, indent=2) - - -def get_outdated_clusters(body: object, xml_clusters: dict) -> list[ClusterInfo]: - result = [] - for endpoint in body.get("endpointTypes", []): - endpoint_id = endpoint.get("id") - 1 - for cluster in endpoint.get("clusters", []): - for attribute in cluster.get("attributes", []): - if attribute.get("name") != "ClusterRevision" or attribute.get("storageOption") != "RAM": - continue - cluster_revision = int(attribute.get("defaultValue")) - spec_revision = xml_clusters[cluster.get("code")].revision - # Filter in outdated clusters only - if (cluster_revision == spec_revision): - break - cluster_info = ClusterInfo(endpoint_id=endpoint_id, cluster_code=cluster.get("code"), - cluster_spec_revision=spec_revision, cluster_name=cluster.get("name"), json_attribute=attribute) - result.append(cluster_info) - return result - - -def main(): - parser = argparse.ArgumentParser( - description="Process ZAP Files and update outdated cluster revisions according to the spec") - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("--print-only", action="store_true", help="Print outdated cluster information.") - group.add_argument("--update", action="store_true", help="Update outdated cluster revisions on ZAP file.") - parser.add_argument("filenames", nargs="+", help="A sequence of ZAP filenames.") - args = parser.parse_args() - - print("**WARNING**: This tool only updates the revision number. Please ensure any new attributes, events or commands are implemented accordingly.") - outdated_count = 0 - for zap_filename in args.filenames: - print("Checking for outdated cluster revisions on: %s" % zap_filename) - body = load_zap(zap_filename) - spec_xml_clusters, problems = build_xml_clusters() - - outdated_clusters = get_outdated_clusters(body, spec_xml_clusters) - print("%d found!" % len(outdated_clusters)) - outdated_count += len(outdated_clusters) - print(*outdated_clusters, sep='\n') - - if args.print_only: - continue - - # Update outdated cluster revisions according to the spec - for cluster in outdated_clusters: - cluster.update_cluster_revision() - if (outdated_clusters): - print('Cluster revisions updated successfully!\n') - - # Check there's no longer any outdated cluster - assert (not get_outdated_clusters(body, spec_xml_clusters)) - - save_zap(body, zap_filename) - - # If it's printing only, return the number of outdated clusters, so it can be used as a test - if args.print_only: - return outdated_count - - -if __name__ == "__main__": - main() From d2aa9a674ca9d988ab97e403a3ea3c680f832e8c Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 25 Nov 2024 18:13:04 +0000 Subject: [PATCH 08/13] Restyled by prettier-markdown --- scripts/tools/zap/README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/tools/zap/README.md b/scripts/tools/zap/README.md index 8f29fa7cd37902..03512dec8f6cab 100644 --- a/scripts/tools/zap/README.md +++ b/scripts/tools/zap/README.md @@ -6,7 +6,6 @@ orphan: true This directory contains various tools related to ZAP. - # ZAP Cluster Revision Update Tool (update_cluster_revisions.py) This tool parses all ZAP files in the codebase and updates cluster revisions. @@ -49,12 +48,16 @@ options: ``` [Note] -* Use `--dry-run` to print only, don't update ZAP files -* Omit `--cluster-id` to search for all clusters -* Omit `--new-revision` to update to the latest revision according to the specification -* Optionally provide `--old-revision`, `--cluster-id` and `--new-revision` to update only clusters that match `old-revision`. -Example #1: Check all outdated cluster revisions and print only (do not modify the ZAP file `--dry-run`): +- Use `--dry-run` to print only, don't update ZAP files +- Omit `--cluster-id` to search for all clusters +- Omit `--new-revision` to update to the latest revision according to the + specification +- Optionally provide `--old-revision`, `--cluster-id` and `--new-revision` to + update only clusters that match `old-revision`. + +Example #1: Check all outdated cluster revisions and print only (do not modify +the ZAP file `--dry-run`): ``` ./scripts/tools/zap/update_cluster_revisions.py --dry-run --no-parallel @@ -105,8 +108,8 @@ Endpoint: 1 cluster_code: 1030 cluster_revision: 4 cluster_spec_revision: 5 name ... ``` -Example #2: Check for possible outdated revisions of the cluster 0x28 revisions and -print only (do not modify the ZAP file): +Example #2: Check for possible outdated revisions of the cluster 0x28 revisions +and print only (do not modify the ZAP file): ``` ./scripts/tools/zap/update_cluster_revisions.py --dry-run --no-parallel --cluster-id 0x28 @@ -135,10 +138,13 @@ Endpoint: 1 cluster_code: 0x28 cluster_revision: 2 cluster_spec_revision: 4 name ... ``` -Example #3: Update outdated revisions of the cluster 0x28 to the latest in the specification: +Example #3: Update outdated revisions of the cluster 0x28 to the latest in the +specification: + ``` ./scripts/tools/zap/update_cluster_revisions.py --no-parallel --cluster-id 0x28 ``` + Example output: ``` @@ -159,13 +165,13 @@ Searching for zcl file from /usr/local/google/home/sergiosoares/connectedhomeip/ 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -Application is failing the Device Type Specification as follows: +Application is failing the Device Type Specification as follows: - ⚠ Check Device Type Compliance on endpoint: 0, device type: MA-rootdevice, cluster: Localization Configuration, attribute: ActiveLocale needs to be enabled - ⚠ Check Device Type Compliance on endpoint: 0, device type: MA-rootdevice, cluster: Localization Configuration, attribute: SupportedLocales needs to be enabled - ⚠ Check Device Type Compliance on endpoint: 2, device type: MA-windowcovering, cluster: Window Covering server needs bit 3 enabled in the Feature Map attribute -Application is failing the Cluster Specification as follows: +Application is failing the Cluster Specification as follows: 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 From c9ece3cbb1e3eca6479b2ddbb2df291d76739f6b Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 25 Nov 2024 18:13:10 +0000 Subject: [PATCH 09/13] Restyled by isort --- scripts/tools/zap/update_cluster_revisions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/tools/zap/update_cluster_revisions.py b/scripts/tools/zap/update_cluster_revisions.py index 5fe96e2e9846e2..c8faad59fb6732 100755 --- a/scripts/tools/zap/update_cluster_revisions.py +++ b/scripts/tools/zap/update_cluster_revisions.py @@ -22,8 +22,9 @@ import os import subprocess import sys -from pathlib import Path from dataclasses import dataclass +from pathlib import Path + from chip.testing.spec_parsing import build_xml_clusters BASIC_INFORMATION_CLUSTER_ID = int("0x0039", 16) From 44691e6ed96ae04079ba00f319e0277f55d2a9e6 Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Tue, 26 Nov 2024 21:40:13 +0000 Subject: [PATCH 10/13] fix python lint error --- scripts/tools/zap/update_cluster_revisions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tools/zap/update_cluster_revisions.py b/scripts/tools/zap/update_cluster_revisions.py index c8faad59fb6732..bab77b16039c4a 100755 --- a/scripts/tools/zap/update_cluster_revisions.py +++ b/scripts/tools/zap/update_cluster_revisions.py @@ -86,7 +86,7 @@ def get_outdated_clusters(data: object, xml_clusters: dict, args) -> list[Cluste try: cluster_revision = int(attribute.get("defaultValue")) spec_revision = xml_clusters[cluster.get("code")].revision - except: + except (KeyError, ValueError): continue # Filter in outdated clusters only if (cluster_revision == spec_revision): From d1de25becc67a7a251ce897ebff97f402596bd43 Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Thu, 12 Dec 2024 17:39:24 +0000 Subject: [PATCH 11/13] Avoid requiring the python env if the user provides --new-revision --- scripts/tools/zap/README.md | 4 +++- scripts/tools/zap/update_cluster_revisions.py | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/tools/zap/README.md b/scripts/tools/zap/README.md index 03512dec8f6cab..66a59cbb32f6cf 100644 --- a/scripts/tools/zap/README.md +++ b/scripts/tools/zap/README.md @@ -15,7 +15,9 @@ attributes, events or commands are implemented accordingly. ## Prerequisites -This tool uses the python environment, which can be built and activated using: +This tool may require the python environment to parse the latest version of the +cluster revisions (i.e. if you don't provide a '--new-revision' argument). The python +environment can be built and activated using: ``` # Build diff --git a/scripts/tools/zap/update_cluster_revisions.py b/scripts/tools/zap/update_cluster_revisions.py index bab77b16039c4a..3a0041ec54dc39 100755 --- a/scripts/tools/zap/update_cluster_revisions.py +++ b/scripts/tools/zap/update_cluster_revisions.py @@ -25,8 +25,6 @@ from dataclasses import dataclass from pathlib import Path -from chip.testing.spec_parsing import build_xml_clusters - BASIC_INFORMATION_CLUSTER_ID = int("0x0039", 16) CHIP_ROOT_DIR = os.path.realpath( os.path.join(os.path.dirname(__file__), '../../..')) @@ -222,10 +220,20 @@ def main(): os.chdir(CHIP_ROOT_DIR) targets = getTargets(args.cluster_id if args.cluster_id else 0) - spec_xml_clusters, problems = build_xml_clusters() - items = [(args, target, spec_xml_clusters) for target in targets] - update_func = updateOne if args.new_revision else updateOneToLatest + if args.new_revision: + update_func = updateOne + items = [(args, target, None) for target in targets] + else: + update_func = updateOneToLatest + try: + from chip.testing.spec_parsing import build_xml_clusters + except ImportError: + print("Couldn't import 'chip.testing.spec_parsing'. Try building/activating your python environment: ./scripts/build_python.sh -i out/python_env && source out/python_env/bin/activate)") + return 1 + spec_xml_clusters, problems = build_xml_clusters() + items = [(args, target, spec_xml_clusters) for target in targets] + if args.parallel: # Ensure each zap run is independent os.environ['ZAP_TEMPSTATE'] = '1' From 145c86c59c3f6d83c289da74ef9ff07a37f8b62c Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 12 Dec 2024 17:40:32 +0000 Subject: [PATCH 12/13] Restyled by prettier-markdown --- scripts/tools/zap/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tools/zap/README.md b/scripts/tools/zap/README.md index 66a59cbb32f6cf..a2a09144bc3556 100644 --- a/scripts/tools/zap/README.md +++ b/scripts/tools/zap/README.md @@ -16,8 +16,8 @@ attributes, events or commands are implemented accordingly. ## Prerequisites This tool may require the python environment to parse the latest version of the -cluster revisions (i.e. if you don't provide a '--new-revision' argument). The python -environment can be built and activated using: +cluster revisions (i.e. if you don't provide a '--new-revision' argument). The +python environment can be built and activated using: ``` # Build From fe739539520c3bb18f1d41c6dc715ffeb36688df Mon Sep 17 00:00:00 2001 From: Sergio Soares Date: Mon, 16 Dec 2024 11:42:03 -0500 Subject: [PATCH 13/13] Accept boris' review suggestion Co-authored-by: Boris Zbarsky --- scripts/tools/zap/update_cluster_revisions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tools/zap/update_cluster_revisions.py b/scripts/tools/zap/update_cluster_revisions.py index 3a0041ec54dc39..9d1cde12b33e1c 100755 --- a/scripts/tools/zap/update_cluster_revisions.py +++ b/scripts/tools/zap/update_cluster_revisions.py @@ -79,7 +79,7 @@ def get_outdated_clusters(data: object, xml_clusters: dict, args) -> list[Cluste if args.cluster_id is not None and cluster['code'] != args.cluster_id: continue for attribute in cluster.get("attributes", []): - if attribute.get("name") != "ClusterRevision" or attribute.get("storageOption") != "RAM": + if attribute.get("name") != "ClusterRevision" or attribute.get("storageOption") == "External": continue try: cluster_revision = int(attribute.get("defaultValue"))