[uss_qualifier] Add report evaluation to uss_qualifier (interuss#257)

* Add report validation to uss_qualifier

* Fix missing resource in dss_probing dev configuration

* Fix lint

monitoring/uss_qualifier/configurations/configuration.py

@@ -3,9 +3,10 @@
 from implicitdict import ImplicitDict
 
 from monitoring.monitorlib.dicts import JSONAddress
-from monitoring.uss_qualifier.fileio import load_dict_with_references
+from monitoring.uss_qualifier.reports.validation.definitions import (
+    ValidationConfiguration,
+)
 from monitoring.uss_qualifier.requirements.definitions import RequirementCollection
-from monitoring.uss_qualifier.requirements.documentation import RequirementSetID
 from monitoring.uss_qualifier.resources.definitions import ResourceCollection
 from monitoring.uss_qualifier.suites.definitions import (
     TestSuiteActionDeclaration,
@@ -118,6 +119,9 @@ class USSQualifierConfigurationV1(ImplicitDict):
     artifacts: Optional[ArtifactsConfiguration] = None
     """If specified, configuration describing the artifacts related to the test run"""
 
+    validation: Optional[ValidationConfiguration] = None
+    """If specified, configuration describing how to validate the output report (and return an error code if validation fails)"""
+
 
 class USSQualifierConfiguration(ImplicitDict):
     v1: Optional[USSQualifierConfigurationV1]

monitoring/uss_qualifier/configurations/dev/dss_probing.yaml

@@ -64,6 +64,18 @@ v1:
             reference_time: '2023-01-10T00:00:00.123456+00:00'
             time_start: '2023-01-10T00:00:01.123456+00:00'
             time_end: '2023-01-10T01:00:01.123456+00:00'
+        problematically_big_area:
+          resource_type: resources.VerticesResource
+          specification:
+            vertices:
+              - lat: 38
+                lng: -81
+              - lat: 37
+                lng: -81
+              - lat: 37
+                lng: -80
+              - lat: 38
+                lng: -80
     action:
       test_suite:
         suite_type: suites.interuss.dss.all_tests
@@ -72,6 +84,7 @@ v1:
           f3411v22a_dss_instances: netrid_dss_instances_v22a
           id_generator: id_generator
           service_area: service_area
+          problematically_big_area: problematically_big_area
   artifacts:
     report:
       report_path: output/report_dss_probing.json
@@ -86,3 +99,7 @@ v1:
       participant_requirements:
         uss1: all_astm_dss_requirements
         uss2: all_astm_dss_requirements
+  validation:
+    criteria:
+      - full_success: {}
+      - no_skipped_actions: {}

monitoring/uss_qualifier/configurations/dev/f3548.yaml

@@ -28,3 +28,5 @@ v1:
       participant_requirements:
         uss1: scd
         uss2: scd
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml

@@ -13,3 +13,5 @@ v1:
   artifacts:
     report:
       report_path: output/report_f3548_flight_intent_validation.json
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml

@@ -14,3 +14,5 @@ v1:
   artifacts:
     report:
       report_path: output/report_f3548_nominal_planning_conflict_equal_priority_not_permitted.json
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml

@@ -14,3 +14,5 @@ v1:
   artifacts:
     report:
       report_path: output/report_f3548_nominal_planning_conflict_higher_priority.json
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml

@@ -12,4 +12,6 @@ v1:
         on_failure: Abort
   artifacts:
     report:
-      report_path: output/report_general_flight_auth.json
+      report_path: output/report_general_flight_auth.json
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.json

@@ -85,6 +85,9 @@
           }
         }
       }
+    },
+    "validation": {
+      "$ref": "./library/validation.yaml#/normal_test"
     }
   }
 }

monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml

@@ -17,3 +17,5 @@ v1:
       report_path: output/report_geoawareness_cis.json
     graph:
       gv_path: output/report_geoawareness_cis.gv
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml

@@ -12,4 +12,6 @@ v1:
         on_failure: Abort
   artifacts:
     report:
-      report_path: output/report_geospatial_comprehension.json
+      report_path: output/report_geospatial_comprehension.json
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/library/validation.yaml

@@ -0,0 +1,5 @@
+normal_test:
+  $content_schema: monitoring/uss_qualifier/reports/validation/report_validation/ValidationConfiguration.json
+  criteria:
+    - full_success: {}
+    - no_skipped_actions: {}

monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml

@@ -21,3 +21,5 @@ v1:
             report_path: output/report_netrid_v19.json
         tested_roles:
             report_path: output/tested_roles_netrid_v19
+    validation:
+      $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml

@@ -37,3 +37,5 @@ v1:
             participant_requirements:
                 uss1: sp_dp_dss
                 uss2: sp_dss
+    validation:
+      $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/noop.yaml

@@ -26,4 +26,6 @@ v1:
   artifacts:
     report:
       # Path to main report output
-      report_path: output/report_noop.json
+      report_path: output/report_noop.json
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/uspace.yaml

@@ -89,3 +89,5 @@ v1:
         uss2: uspace
     sequence_view:
       output_path: output/sequence_uspace
+  validation:
+    $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/main.py

@@ -16,6 +16,7 @@
     USSQualifierConfiguration,
     ArtifactsConfiguration,
     ReportConfiguration,
+    USSQualifierConfigurationV1,
 )
 from monitoring.uss_qualifier.fileio import load_dict_with_references
 from monitoring.uss_qualifier.reports.documents import make_report_html
@@ -27,6 +28,9 @@
 from monitoring.uss_qualifier.reports.graphs import make_graph
 from monitoring.uss_qualifier.reports.report import TestRunReport, redact_access_tokens
 from monitoring.uss_qualifier.reports.templates import render_templates
+from monitoring.uss_qualifier.reports.validation.report_validation import (
+    validate_report,
+)
 from monitoring.uss_qualifier.resources.resource import create_resources
 from monitoring.uss_qualifier.signatures import (
     compute_signature,
@@ -145,7 +149,7 @@ def main() -> int:
         logger.info("Exiting because --exit-before-execution specified.")
         return os.EX_OK
 
-    config = whole_config.v1
+    config: USSQualifierConfigurationV1 = whole_config.v1
     if args.report:
         if not config.artifacts:
             config.artifacts = ArtifactsConfiguration(
@@ -210,6 +214,14 @@ def main() -> int:
             logger.info(f"Writing sequence view to {path}")
             generate_sequence_view(report, config.artifacts.sequence_view)
 
+    if "validation" in config and config.validation:
+        logger.info(f"Validating test run report for configuration '{args.config}'")
+        if not validate_report(report, config.validation):
+            logger.error(
+                f"Validation failed on test run report for configuration '{args.config}'"
+            )
+            return -1
+
     return os.EX_OK
 
 

monitoring/uss_qualifier/reports/validation/definitions.py

@@ -0,0 +1,28 @@
+from typing import Optional, List
+
+from implicitdict import ImplicitDict
+
+
+class FullSuccessCriterion(ImplicitDict):
+    """Validation criterion that every element of the report must indicate success."""
+
+    pass
+
+
+class NoSkippedActionsCriterion(ImplicitDict):
+    """Validation criterion that no actions in the entire test run may be skipped."""
+
+    pass
+
+
+class ValidationCriterion(ImplicitDict):
+    """Wrapper for all the potential types of validation criteria."""
+
+    full_success: Optional[FullSuccessCriterion] = None
+    no_skipped_actions: Optional[NoSkippedActionsCriterion] = None
+
+
+class ValidationConfiguration(ImplicitDict):
+    """Complete set of validation criteria that a test run report must satisfy."""
+
+    criteria: List[ValidationCriterion]

monitoring/uss_qualifier/reports/validation/report_validation.py

@@ -0,0 +1,103 @@
+from loguru import logger
+
+from monitoring.monitorlib.dicts import JSONAddress
+from monitoring.uss_qualifier.reports.report import TestRunReport, TestSuiteActionReport
+from monitoring.uss_qualifier.reports.validation.definitions import (
+    ValidationConfiguration,
+)
+
+
+def _validate_action_full_success(
+    report: TestSuiteActionReport, context: JSONAddress
+) -> bool:
+    test_suite, test_scenario, action_generator = report.get_applicable_report()
+    if test_scenario:
+        success = report.test_scenario.successful
+        if not success:
+            logger.error(
+                f"Full success not achieved because {context}.test_scenario.successful was False"
+            )
+    elif test_suite:
+        if report.test_suite.successful:
+            success = True
+        else:
+            success = False
+            logger.error(
+                f"Full success not achieved because {context}.test_suite.successful was False"
+            )
+        for i, a in enumerate(report.test_suite.actions):
+            success = success and _validate_action_full_success(
+                a, JSONAddress(context + f".test_suite.actions[{i}]")
+            )
+    elif action_generator:
+        if report.action_generator.successful:
+            success = True
+        else:
+            success = False
+            logger.error(
+                f"Full success not achieved because {context}.action_generator.successful was False"
+            )
+        for i, a in enumerate(report.action_generator.actions):
+            success = success and _validate_action_full_success(
+                a, JSONAddress(context + f".action_generator.actions[{i}]")
+            )
+    return success
+
+
+def _validate_full_success(report: TestRunReport) -> bool:
+    return _validate_action_full_success(report.report, JSONAddress("$"))
+
+
+def _validate_action_no_skipped_actions(
+    report: TestSuiteActionReport, context: JSONAddress
+) -> bool:
+    test_suite, test_scenario, action_generator = report.get_applicable_report()
+    if test_scenario:
+        success = True
+    elif test_suite:
+        if (
+            "skipped_actions" not in report.test_suite
+            or not report.test_suite.skipped_actions
+        ):
+            success = True
+        else:
+            success = False
+            for sa in report.test_suite.skipped_actions:
+                logger.error(
+                    f"No skipped actions not achieved because {context}.test_suite had a skipped action for action index {sa.action_declaration_index}: {sa.reason}"
+                )
+        for i, a in enumerate(report.test_suite.actions):
+            success = success and _validate_action_no_skipped_actions(
+                a, JSONAddress(context + f".test_suite.actions[{i}]")
+            )
+    elif action_generator:
+        success = True
+        for i, a in enumerate(report.action_generator.actions):
+            success = success and _validate_action_no_skipped_actions(
+                a, JSONAddress(context + f".action_generator.actions[{i}]")
+            )
+    return success
+
+
+def _validate_no_skipped_actions(report: TestRunReport) -> bool:
+    return _validate_action_no_skipped_actions(report.report, JSONAddress("$"))
+
+
+def validate_report(report: TestRunReport, validation: ValidationConfiguration) -> bool:
+    """Validate that the provided report meets all the specified validation criteria.
+
+    Validation failures are logged as errors.
+
+    Args:
+        report: Report to validate.
+        validation: Validation criteria.
+
+    Returns: True if the report satisfies all criteria, False otherwise.
+    """
+    success = True
+    for criterion in validation.criteria:
+        if criterion.full_success is not None:
+            success = success and _validate_full_success(report)
+        if criterion.no_skipped_actions is not None:
+            success = success and _validate_no_skipped_actions(report)
+    return success

monitoring/uss_qualifier/run_locally.sh

@@ -60,7 +60,6 @@ else
     docker_args="-it"
   fi
 
-  start_time=$(date +%Y-%m-%dT%H:%M:%S)
   echo "========== Running uss_qualifier for configuration ${CONFIG_NAME} =========="
   # shellcheck disable=SC2086
   docker run ${docker_args} --name uss_qualifier \
@@ -77,18 +76,5 @@ else
     interuss/monitoring \
     python main.py $QUALIFIER_OPTIONS
   echo "========== Completed uss_qualifier for configuration ${CONFIG_NAME} =========="
-
-  # Set return code according to whether the test run was fully successful
-  reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time")
-  # shellcheck disable=SC2068
-  for REPORT in ${reports_generated[@]}; do
-    successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT")
-    if echo "${successful}" | grep -iqF true; then
-      echo "Full success indicated by $REPORT"
-    else
-      echo "Could not establish that all uss_qualifier tests passed in $REPORT"
-      exit 1
-    fi
-  done
 fi
 

monitoring/uss_qualifier/suites/interuss/dss/all_tests.yaml

@@ -4,13 +4,15 @@ resources:
   f3411v22a_dss_instances: resources.astm.f3411.DSSInstancesResource
   id_generator: resources.interuss.IDGeneratorResource
   service_area: resources.netrid.ServiceAreaResource
+  problematically_big_area: resources.VerticesResource
 actions:
   - action_generator:
       generator_type: action_generators.astm.f3411.ForEachDSS
       resources:
         dss_instances: f3411v19_dss_instances
         id_generator: id_generator
         service_area: service_area
+        problematically_big_area: problematically_big_area
       specification:
         action_to_repeat:
           test_suite:
@@ -20,6 +22,7 @@ actions:
               all_dss_instances: dss_instances
               id_generator: id_generator
               isa: service_area
+              problematically_big_area: problematically_big_area
           on_failure: Continue
         dss_instances_source: dss_instances
         dss_instance_id: dss
@@ -30,6 +33,7 @@ actions:
         dss_instances: f3411v22a_dss_instances
         id_generator: id_generator
         service_area: service_area
+        problematically_big_area: problematically_big_area
       specification:
         action_to_repeat:
           test_suite:
@@ -39,6 +43,7 @@ actions:
               all_dss_instances: dss_instances
               id_generator: id_generator
               isa: service_area
+              problematically_big_area: problematically_big_area
           on_failure: Continue
         dss_instances_source: dss_instances
         dss_instance_id: dss