[uss_qualifier] Add runtime metadata (#728)

* Add runtime metadata

* Fix default logic

monitoring/uss_qualifier/main.py

@@ -67,6 +67,12 @@ def parseArgs() -> argparse.Namespace:
         help="Path to folder where artifacts should be written.  If not specified, defaults to output/{CONFIG_NAME}",
     )
 
+    parser.add_argument(
+        "--runtime-metadata",
+        default=None,
+        help="JSON string containing runtime metadata to record in the test run report (if specified).",
+    )
+
     return parser.parse_args()
 
 
@@ -138,6 +144,7 @@ def run_config(
     skip_validation: bool,
     exit_before_execution: bool,
     output_path: Optional[str],
+    runtime_metadata: Optional[dict],
 ):
     config_src = load_dict_with_references(config_name)
 
@@ -193,6 +200,9 @@ def run_config(
     logger.info("Executing test run")
     report = execute_test_run(whole_config, description)
 
+    if runtime_metadata is not None:
+        report.runtime_metadata = runtime_metadata
+
     if config.artifacts:
         generate_artifacts(report, config.artifacts, output_path)
 
@@ -210,6 +220,12 @@ def run_config(
 def main() -> int:
     args = parseArgs()
 
+    runtime_metadata = (
+        json.loads(args.runtime_metadata) if args.runtime_metadata else None
+    )
+    if runtime_metadata is not None and not isinstance(runtime_metadata, dict):
+        raise ValueError("--runtime-metadata must specify a JSON dictionary")
+
     config_names = str(args.config).split(",")
 
     if args.config_output:
@@ -235,6 +251,7 @@ def main() -> int:
             args.skip_validation,
             args.exit_before_execution,
             output_path,
+            runtime_metadata,
         )
         if exit_code != os.EX_OK:
             return exit_code

monitoring/uss_qualifier/reports/report.py

@@ -738,6 +738,9 @@ class TestRunReport(ImplicitDict):
     report: TestSuiteActionReport
     """Report produced by configured test action"""
 
+    runtime_metadata: Optional[dict]
+    """Metadata for the test run specified at runtime."""
+
 
 def redact_access_tokens(report: Union[Dict[str, Any], list]) -> None:
     if isinstance(report, dict):

monitoring/uss_qualifier/reports/templates/sequence_view/overview.html

@@ -72,6 +72,16 @@ <h1>Sequence view of uss_qualifier test run</h1>
   </div>
 </div>
 
+{% if "runtime_metadata" in report %}
+<div id="runtime_metadata_section" class="collapseable">
+  <div class="node_key" onclick="showHide(document.getElementById('runtime_metadata_section'))"><h2>Runtime metadata</h2></div>
+  <div class="node_value">
+    {{ explorer_content("runtime_metadata_tree", report.runtime_metadata) }}
+    {% set collapsible.queries = collapsible.queries + ["runtime_metadata_tree"] %}
+  </div>
+</div>
+{% endif %}
+
 <div>
   <h2>Scenarios executed</h2>
   <table>
@@ -147,6 +157,9 @@ <h2>Scenarios executed</h2>
 <script>
   showHide(document.getElementById('resources_configuration_section'));
   showHide(document.getElementById('full_configuration_section'));
+  {% if "runtime_metadata" in report %}
+    showHide(document.getElementById('runtime_metadata_section'));
+  {% endif %}
 </script>
 </body>
 </html>

schemas/monitoring/uss_qualifier/reports/report/TestRunReport.json

@@ -30,6 +30,13 @@
     "report": {
       "$ref": "TestSuiteActionReport.json",
       "description": "Report produced by configured test action"
+    },
+    "runtime_metadata": {
+      "description": "Metadata for the test run specified at runtime.",
+      "type": [
+        "object",
+        "null"
+      ]
     }
   },
   "required": [