Actually enable our YAML tests in the python-calls-chip-tool runner.

We were now getting the right test list, but for each test we were not finding
the test file.  And the harness did not treat that as an error.

scripts/tests/chiptest/__init__.py

@@ -243,7 +243,10 @@ def tests_with_command(chip_tool: str, is_manual: bool):
         )
 
 
-def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
+def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
+    """
+    use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
+    """
     manual_tests = _GetManualTests()
     flaky_tests = _GetFlakyTests()
     slow_tests = _GetSlowTests()
@@ -270,21 +273,26 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
         if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests:
             tags.add(TestTag.IN_DEVELOPMENT)
 
+        if use_short_run_name:
+            run_name = path.stem  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
+        else:
+            run_name = str(path)
+
         yield TestDefinition(
-            run_name=str(path),
+            run_name=run_name,
             name=path.stem,  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
             target=target_for_name(path.name),
             tags=tags,
         )
 
 
 def AllReplYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
         yield test
 
 
 def AllChipToolYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
         yield test
 
 

scripts/tests/yaml/runner.py

@@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration
     tests_finder = TestsFinder(configuration_directory, configuration_name)
 
     parser_config = TestParserConfig(pics, specifications, kwargs)
-    parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger())
+    test_list = tests_finder.get(test_name)
+    if len(test_list) == 0:
+        raise Exception(f"No tests found for test name '{test_name}'")
+    parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger())
     parser_builder_config.options.stop_on_error = stop_on_error
     while ctx:
         ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters)

src/app/tests/suites/TestClusterMultiFabric.yaml

@@ -328,39 +328,27 @@ tests:
                   },
                   {
                       FabricIndex: 2,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
                   {
                       FabricIndex: 2,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
               ]
 
@@ -375,39 +363,27 @@ tests:
               [
                   {
                       FabricIndex: 1,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
                   {
                       FabricIndex: 1,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
                   {
                       FabricIndex: 2,