From 45e7ad9ac0afec9e106a1f12ed85ef994dfc9929 Mon Sep 17 00:00:00 2001 From: Boris Zbarsky Date: Wed, 6 Sep 2023 12:14:48 -0400 Subject: [PATCH] Actually enable our YAML tests in the python-calls-chip-tool runner. We were now getting the right test list, but for each test we were not finding the test file. And the harness did not treat that as an error. --- scripts/tests/chiptest/__init__.py | 16 +++- scripts/tests/yaml/runner.py | 5 +- .../tests/suites/TestClusterMultiFabric.yaml | 88 +++++++------------ 3 files changed, 48 insertions(+), 61 deletions(-) diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index ce312d492beabd..3bd1a1a95df1a4 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -243,7 +243,10 @@ def tests_with_command(chip_tool: str, is_manual: bool): ) -def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): +def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool): + """ + use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml" + """ manual_tests = _GetManualTests() flaky_tests = _GetFlakyTests() slow_tests = _GetSlowTests() @@ -270,8 +273,13 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests: tags.add(TestTag.IN_DEVELOPMENT) + if use_short_run_name: + run_name = path.stem # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" + else: + run_name = str(path) + yield TestDefinition( - run_name=str(path), + run_name=run_name, name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" target=target_for_name(path.name), tags=tags, @@ -279,12 +287,12 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): def AllReplYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False): yield test def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True): yield test diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py index 04f30401eb1b02..04a1f9453d3b98 100755 --- a/scripts/tests/yaml/runner.py +++ b/scripts/tests/yaml/runner.py @@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration tests_finder = TestsFinder(configuration_directory, configuration_name) parser_config = TestParserConfig(pics, specifications, kwargs) - parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger()) + test_list = tests_finder.get(test_name) + if len(test_list) == 0: + raise Exception(f"No tests found for test name '{test_name}'") + parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger()) parser_builder_config.options.stop_on_error = stop_on_error while ctx: ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters) diff --git a/src/app/tests/suites/TestClusterMultiFabric.yaml b/src/app/tests/suites/TestClusterMultiFabric.yaml index 97b38ee103df85..c266e66fb23ff5 100644 --- a/src/app/tests/suites/TestClusterMultiFabric.yaml +++ b/src/app/tests/suites/TestClusterMultiFabric.yaml @@ -328,39 +328,27 @@ tests: }, { FabricIndex: 2, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 2, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, ] @@ -375,39 +363,27 @@ tests: [ { FabricIndex: 1, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 1, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 2,