[uss_qualifier] Enable execution of multiple configurations in the sa…

…me container (interuss#266)

monitoring/uss_qualifier/README.md

@@ -10,6 +10,8 @@ The `uss_qualifier` tool is a synchronous executable built into the `interuss/mo
 
 The primary input accepted by uss_qualifier is the "configuration" specified with the `--config` option.  This option should be a [reference to a configuration file](configurations/README.md) that the user has constructed or been provided to test the desired system for the desired characteristics.  If testing a standard local system (DSS + dummy auth + mock USSs), the user can specify an alternate configuration reference as a single argument to `run_locally.sh` (the default configuration is `configurations.dev.local_test`).
 
+Several comma-separated "configurations" may be passed via the `--config` option.  If specified, the `--report` and `--config-output` options must have the same number of comma-separated values, which may be empty.
+
 When building a custom configuration file, consider starting from [`configurations.dev.f3548_self_contained`](configurations/dev/f3548_self_contained.yaml), as it contains all information necessary to run the test without the usage of sometimes-configuring `$ref`s and `allOf`s.  See [configurations documentation](configurations/README.md) for more information.
 
 ### Quick start

monitoring/uss_qualifier/main.py

@@ -45,20 +45,20 @@ def parseArgs() -> argparse.Namespace:
 
     parser.add_argument(
         "--config",
-        help="Configuration string according to monitoring/uss_qualifier/configurations/README.md",
+        help="Configuration string according to monitoring/uss_qualifier/configurations/README.md; Several comma-separated strings may be specified",
         required=True,
     )
 
     parser.add_argument(
         "--report",
         default=None,
-        help="(Overrides setting in artifacts configuration) File name of the report to write (if test configuration provided) or read (if test configuration not provided)",
+        help="(Overrides setting in artifacts configuration) File name of the report to write (if test configuration provided) or read (if test configuration not provided); Several comma-separated file names matching the configurations may be specified",
     )
 
     parser.add_argument(
         "--config-output",
         default=None,
-        help="If specified, write the configuration as parsed (potentially from multiple files) to the single file specified by this path",
+        help="If specified, write the configuration as parsed (potentially from multiple files) to the single file specified by this path; Several comma-separated file names matching the configurations may be specified",
     )
 
     parser.add_argument(
@@ -115,12 +115,16 @@ def execute_test_run(
     )
 
 
-def main() -> int:
-    args = parseArgs()
-
-    config_src = load_dict_with_references(args.config)
+def run_config(
+    config_name: str,
+    config_output: str,
+    report_path: str,
+    skip_validation: bool,
+    exit_before_execution: bool,
+):
+    config_src = load_dict_with_references(config_name)
 
-    if not args.skip_validation:
+    if not skip_validation:
         logger.info("Validating configuration...")
         validation_errors = validate_config(config_src)
         if validation_errors:
@@ -132,33 +136,33 @@ def main() -> int:
 
     whole_config = ImplicitDict.parse(config_src, USSQualifierConfiguration)
 
-    if args.config_output:
-        logger.info("Writing flattened configuration to {}", args.config_output)
-        if args.config_output.lower().endswith(".json"):
-            with open(args.config_output, "w") as f:
+    if config_output:
+        logger.info("Writing flattened configuration to {}", config_output)
+        if config_output.lower().endswith(".json"):
+            with open(config_output, "w") as f:
                 json.dump(whole_config, f, indent=2, sort_keys=True)
-        elif args.config_output.lower().endswith(".yaml"):
-            with open(args.config_output, "w") as f:
+        elif config_output.lower().endswith(".yaml"):
+            with open(config_output, "w") as f:
                 yaml.dump(json.loads(json.dumps(whole_config)), f, sort_keys=True)
         else:
             raise ValueError(
                 "Unsupported extension for --config-output; only .json or .yaml file paths may be specified"
             )
 
-    if args.exit_before_execution:
+    if exit_before_execution:
         logger.info("Exiting because --exit-before-execution specified.")
-        return os.EX_OK
+        return
 
     config: USSQualifierConfigurationV1 = whole_config.v1
-    if args.report:
+    if report_path:
         if not config.artifacts:
             config.artifacts = ArtifactsConfiguration(
-                ReportConfiguration(report_path=args.report)
+                ReportConfiguration(report_path=report_path)
             )
         elif not config.artifacts.report:
-            config.artifacts.report = ReportConfiguration(report_path=args.report)
+            config.artifacts.report = ReportConfiguration(report_path=report_path)
         else:
-            config.artifacts.report.report_path = args.report
+            config.artifacts.report.report_path = report_path
 
     do_not_save_report = False
     if config.test_run:
@@ -215,15 +219,56 @@ def main() -> int:
             generate_sequence_view(report, config.artifacts.sequence_view)
 
     if "validation" in config and config.validation:
-        logger.info(f"Validating test run report for configuration '{args.config}'")
+        logger.info(f"Validating test run report for configuration '{config_name}'")
         if not validate_report(report, config.validation):
             logger.error(
-                f"Validation failed on test run report for configuration '{args.config}'"
+                f"Validation failed on test run report for configuration '{config_name}'"
             )
             return -1
 
     return os.EX_OK
 
 
+def main() -> int:
+    args = parseArgs()
+
+    config_names = str(args.config).split(",")
+
+    if args.config_output:
+        config_outputs = str(args.config_output).split(",")
+        if len(config_outputs) != len(config_names):
+            raise ValueError(
+                f"Need matching number of config_output, expected {len(config_names)}, got {len(config_outputs)}"
+            )
+    else:
+        config_outputs = ["" for _ in config_names]
+
+    if args.report:
+        report_paths = str(args.report).split(",")
+        if len(report_paths) != len(config_names):
+            raise ValueError(
+                f"Need matching number of report, expected {len(config_names)}, got {len(report_paths)}"
+            )
+    else:
+        report_paths = ["" for _ in config_names]
+
+    for idx, config_name in enumerate(config_names):
+        logger.info(
+            f"========== Running uss_qualifier for configuration {config_name} =========="
+        )
+        run_config(
+            config_name,
+            config_outputs[idx],
+            report_paths[idx],
+            args.skip_validation,
+            args.exit_before_execution,
+        )
+        logger.info(
+            f"========== Completed uss_qualifier for configuration {config_name} =========="
+        )
+
+    return os.EX_OK
+
+
 if __name__ == "__main__":
     sys.exit(main())

monitoring/uss_qualifier/run_locally.sh

@@ -24,57 +24,66 @@ CONFIG_NAME="${1:-ALL}"
 OTHER_ARGS=${@:2}
 
 if [ "$CONFIG_NAME" == "ALL" ]; then
-  declare -a all_configurations=( \
-    "configurations.dev.noop" \
-    "configurations.dev.dss_probing" \
-    "configurations.dev.geoawareness_cis" \
-    "configurations.dev.generate_rid_test_data" \
-    "configurations.dev.geospatial_comprehension" \
-    "configurations.dev.general_flight_auth" \
-    "configurations.dev.f3548" \
-    "configurations.dev.f3548_self_contained" \
-    "configurations.dev.netrid_v22a" \
-    "configurations.dev.uspace" \
-  )
-  # TODO: Add configurations.dev.netrid_v19
-  echo "Running configurations: ${all_configurations[*]}"
-  for configuration_name in "${all_configurations[@]}"; do
-    monitoring/uss_qualifier/run_locally.sh "$configuration_name"
-  done
-else
-  CONFIG_FLAG="--config ${CONFIG_NAME}"
+  CONFIG_NAME="\
+configurations.dev.noop,\
+configurations.dev.dss_probing,\
+configurations.dev.geoawareness_cis,\
+configurations.dev.generate_rid_test_data,\
+configurations.dev.geospatial_comprehension,\
+configurations.dev.general_flight_auth,\
+configurations.dev.f3548,\
+configurations.dev.f3548_self_contained,\
+configurations.dev.netrid_v22a,\
+configurations.dev.uspace"
+fi
+# TODO: Add configurations.dev.netrid_v19
 
-  AUTH_SPEC='DummyOAuth(http://oauth.authority.localutm:8085/token,uss_qualifier)'
+echo "Running configuration(s): ${CONFIG_NAME}"
 
-  QUALIFIER_OPTIONS="$CONFIG_FLAG $OTHER_ARGS"
+CONFIG_FLAG="--config ${CONFIG_NAME}"
 
-  OUTPUT_DIR="monitoring/uss_qualifier/output"
-  mkdir -p "$OUTPUT_DIR"
+AUTH_SPEC='DummyOAuth(http://oauth.authority.localutm:8085/token,uss_qualifier)'
 
-  CACHE_DIR="monitoring/uss_qualifier/.templates_cache"
-  mkdir -p "$CACHE_DIR"
+QUALIFIER_OPTIONS="$CONFIG_FLAG $OTHER_ARGS"
 
-  if [ "$CI" == "true" ]; then
-    docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions)
-  else
-    docker_args="-it"
-  fi
+OUTPUT_DIR="monitoring/uss_qualifier/output"
+mkdir -p "$OUTPUT_DIR"
+
+CACHE_DIR="monitoring/uss_qualifier/.templates_cache"
+mkdir -p "$CACHE_DIR"
 
-  echo "========== Running uss_qualifier for configuration ${CONFIG_NAME} =========="
-  # shellcheck disable=SC2086
-  docker run ${docker_args} --name uss_qualifier \
-    --rm \
-    --network interop_ecosystem_network \
-    -u "$(id -u):$(id -g)" \
-    -e PYTHONBUFFERED=1 \
-    -e AUTH_SPEC=${AUTH_SPEC} \
-    -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \
-    -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
-    -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
-    -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
-    -w /app/monitoring/uss_qualifier \
-    interuss/monitoring \
-    python main.py $QUALIFIER_OPTIONS
-  echo "========== Completed uss_qualifier for configuration ${CONFIG_NAME} =========="
+if [ "$CI" == "true" ]; then
+  docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions)
+else
+  docker_args="-it"
 fi
 
+start_time=$(date +%Y-%m-%dT%H:%M:%S)
+# shellcheck disable=SC2086
+docker run ${docker_args} --name uss_qualifier \
+  --rm \
+  --network interop_ecosystem_network \
+  -u "$(id -u):$(id -g)" \
+  -e PYTHONBUFFERED=1 \
+  -e AUTH_SPEC=${AUTH_SPEC} \
+  -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \
+  -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
+  -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
+  -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
+  -w /app/monitoring/uss_qualifier \
+  interuss/monitoring \
+  python main.py $QUALIFIER_OPTIONS
+
+# Set return code according to whether the test run was fully successful
+reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time")
+# shellcheck disable=SC2068
+for REPORT in ${reports_generated[@]}; do
+  successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT")
+  if echo "${successful}" | grep -iqF true; then
+    echo "Full success indicated by $REPORT"
+  else
+    echo "Could not establish that all uss_qualifier tests passed in $REPORT"
+    exit 1
+  fi
+done
+