[uss_qualifier] Add output path override CLI flag (#353)

Add output path override CLI flag

monitoring/uss_qualifier/configurations/configuration.py

@@ -198,7 +198,7 @@ class RawReportConfiguration(ImplicitDict):
 
 class ArtifactsConfiguration(ImplicitDict):
     output_path: str
-    """Path to folder where artifacts should be written."""
+    """Path to folder where artifacts should be written.  Note that this value may be overridden at runtime without affecting the test baseline."""
 
     raw_report: Optional[RawReportConfiguration] = None
     """Configuration for raw report generation"""

monitoring/uss_qualifier/main.py

@@ -4,6 +4,7 @@
 import json
 import os
 import sys
+from typing import Optional
 
 from implicitdict import ImplicitDict
 from loguru import logger
@@ -57,6 +58,12 @@ def parseArgs() -> argparse.Namespace:
         help="If specified, do not validate the format of the provided configuration.",
     )
 
+    parser.add_argument(
+        "--output-path",
+        default=None,
+        help="If specified, override v1.artifacts.output_path with this value.  Overriding in this way does not change the test baseline.",
+    )
+
     return parser.parse_args()
 
 
@@ -108,6 +115,7 @@ def run_config(
     config_output: str,
     skip_validation: bool,
     exit_before_execution: bool,
+    output_path_override: Optional[str],
 ):
     config_src = load_dict_with_references(config_name)
 
@@ -146,7 +154,7 @@ def run_config(
     report = execute_test_run(whole_config)
 
     if config.artifacts:
-        generate_artifacts(report, config.artifacts)
+        generate_artifacts(report, config.artifacts, output_path_override)
 
     if "validation" in config and config.validation:
         logger.info(f"Validating test run report for configuration '{config_name}'")
@@ -182,6 +190,7 @@ def main() -> int:
             config_outputs[idx],
             args.skip_validation,
             args.exit_before_execution,
+            args.output_path,
         )
         if exit_code != os.EX_OK:
             return exit_code

monitoring/uss_qualifier/reports/artifacts.py

@@ -1,5 +1,6 @@
 import json
 import os
+from typing import Optional
 
 from loguru import logger
 
@@ -14,8 +15,13 @@
 )
 
 
-def generate_artifacts(report: TestRunReport, artifacts: ArtifactsConfiguration):
-    os.makedirs(artifacts.output_path, exist_ok=True)
+def generate_artifacts(
+    report: TestRunReport,
+    artifacts: ArtifactsConfiguration,
+    output_path_override: Optional[str] = None,
+):
+    output_path = output_path_override or artifacts.output_path
+    os.makedirs(output_path, exist_ok=True)
 
     def _should_redact(cfg) -> bool:
         return "redact_access_tokens" in cfg and cfg.redact_access_tokens
@@ -26,7 +32,7 @@ def _should_redact(cfg) -> bool:
 
     if artifacts.raw_report:
         # Raw report
-        path = os.path.join(artifacts.output_path, "report.json")
+        path = os.path.join(output_path, "report.json")
         logger.info(f"Writing raw report to {path}")
         raw_report = artifacts.raw_report
         report_to_write = redacted_report if _should_redact(raw_report) else report
@@ -38,7 +44,7 @@ def _should_redact(cfg) -> bool:
 
     if artifacts.report_html:
         # HTML rendering of raw report
-        path = os.path.join(artifacts.output_path, "report.html")
+        path = os.path.join(output_path, "report.html")
         logger.info(f"Writing HTML report to {path}")
         report_to_write = (
             redacted_report if _should_redact(artifacts.report_html) else report
@@ -49,21 +55,21 @@ def _should_redact(cfg) -> bool:
     if artifacts.templated_reports:
         # Templated reports
         render_templates(
-            artifacts.output_path,
+            output_path,
             artifacts.templated_reports,
             redacted_report,
         )
 
     if artifacts.tested_requirements:
         # Tested requirements view
         for tested_reqs_config in artifacts.tested_requirements:
-            path = os.path.join(artifacts.output_path, tested_reqs_config.report_name)
+            path = os.path.join(output_path, tested_reqs_config.report_name)
             logger.info(f"Writing tested requirements view to {path}")
             generate_tested_requirements(redacted_report, tested_reqs_config, path)
 
     if artifacts.sequence_view:
         # Sequence view
-        path = os.path.join(artifacts.output_path, "sequence")
+        path = os.path.join(output_path, "sequence")
         logger.info(f"Writing sequence view to {path}")
         report_to_write = (
             redacted_report if _should_redact(artifacts.sequence_view) else report

schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json

@@ -8,7 +8,7 @@
       "type": "string"
     },
     "output_path": {
-      "description": "Path to folder where artifacts should be written.",
+      "description": "Path to folder where artifacts should be written.  Note that this value may be overridden at runtime without affecting the test baseline.",
       "type": "string"
     },
     "raw_report": {