Meh... I am not sure I love this approach.

lib/galaxy/tool_util/parser/interface.py

@@ -17,7 +17,10 @@
 )
 
 import packaging.version
-from pydantic import BaseModel
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+)
 from typing_extensions import (
     Literal,
     NotRequired,
@@ -92,7 +95,7 @@ class ToolSourceTestInput(TypedDict):
 TestSourceTestOutputColllection = Any
 
 
-class ToolSourceTest(TypedDict):
+class XmlStyleToolSourceTest(TypedDict):
     inputs: ToolSourceTestInputs
     outputs: ToolSourceTestOutputs
     output_collections: List[TestSourceTestOutputColllection]
@@ -107,10 +110,51 @@ class ToolSourceTest(TypedDict):
     command_version: AssertionList
 
 
+JsonToolTestState = Dict[str, Any]
+JsonToolTestOutputs = Dict[str, Any]
+
+
+class JsonToolSourceTest(TypedDict):
+    doc: NotRequired[Optional[str]] = None
+    job: TypedToolTestState
+    outputs: NotRequired[Optional[TypedToolTestOutputs]] = None
+    stdout: NotRequired[Optional[AssertionList]] = None
+    stderr: NotRequired[Optional[AssertionList]] = None
+    expect_exit_code: NotRequired[Optional[int]] = None
+    expect_failure: NotRequired[bool] = False
+    expect_test_failure: NotRequired[bool] = False
+    maxseconds: NotRequired[Optional[int]] = None
+    # consider improving behavior before merge...
+    expect_num_outputs: NotRequired[Optional[int]]
+    command: NotRequired[Optional[AssertionList]] = None
+    command_version: NotRequired[Optional[AssertionList]] = None
+
+
+ToolSourceTest = Union[XmlStyleToolSourceTest, TypedToolSourceTest]
+
+
 class ToolSourceTests(TypedDict):
     tests: List[ToolSourceTest]
 
 
+class ToolSourceTestModel(BaseModel):
+    doc: Optional[str] = None
+    job: TypedToolTestState
+    outputs: Optional[TypedToolTestOutputs] = None
+    stdout: Optional[AssertionList] = None
+    stderr: Optional[AssertionList] = None
+    expect_exit_code: Optional[int] = None
+    expect_failure: bool = False
+    expect_test_failure: bool = False
+    maxseconds: Optional[int] = None
+    # consider improving behavior before merge...
+    expect_num_outputs: Optional[int]
+    command: Optional[AssertionList] = None
+    command_version: Optional[AssertionList]] = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
 class XrefDict(TypedDict):
     value: str
     reftype: str

lib/galaxy/tool_util/verify/parse.py

@@ -10,10 +10,12 @@
 
 from galaxy.tool_util.parser.interface import (
     InputSource,
+    JsonToolSourceTest,
     ToolSource,
     ToolSourceTest,
     ToolSourceTestInputs,
     ToolSourceTests,
+    XmlStyleToolSourceTest,
 )
 from galaxy.tool_util.parser.util import (
     boolean_is_checked,
@@ -59,6 +61,25 @@ def parse_tool_test_descriptions(
 
 def _description_from_tool_source(
     tool_source: ToolSource, raw_test_dict: ToolSourceTest, test_index: int, tool_guid: Optional[str]
+) -> ToolTestDescription:
+    if "job" in raw_test_dict:
+        test = cast(JsonToolSourceTest, raw_test_dict)
+        return _description_from_json_tool_source(tool_source, test, test_index, tool_guid)
+    else:
+        test = cast(XmlStyleToolSourceTest, raw_test_dict)
+        return _description_from_xml_style_tool_source(tool_source, test, test_index, tool_guid)
+
+
+def _description_from_json_tool_source(
+    tool_source: ToolSource, raw_test_dict: JsonToolSourceTest, test_index: int, tool_guid: Optional[str]
+) -> ToolTestDescription:
+    required_files: RequiredFilesT = []
+    required_data_tables: RequiredDataTablesT = []
+    required_loc_files: RequiredLocFileT = []
+
+
+def _description_from_xml_style_tool_source(
+    tool_source: ToolSource, raw_test_dict: XmlStyleToolSourceTest, test_index: int, tool_guid: Optional[str]
 ) -> ToolTestDescription:
     required_files: RequiredFilesT = []
     required_data_tables: RequiredDataTablesT = []