diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index e653164609..e3c679323e 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -3,9 +3,10 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.dicts import JSONAddress -from monitoring.uss_qualifier.fileio import load_dict_with_references +from monitoring.uss_qualifier.reports.validation.definitions import ( + ValidationConfiguration, +) from monitoring.uss_qualifier.requirements.definitions import RequirementCollection -from monitoring.uss_qualifier.requirements.documentation import RequirementSetID from monitoring.uss_qualifier.resources.definitions import ResourceCollection from monitoring.uss_qualifier.suites.definitions import ( TestSuiteActionDeclaration, @@ -118,6 +119,9 @@ class USSQualifierConfigurationV1(ImplicitDict): artifacts: Optional[ArtifactsConfiguration] = None """If specified, configuration describing the artifacts related to the test run""" + validation: Optional[ValidationConfiguration] = None + """If specified, configuration describing how to validate the output report (and return an error code if validation fails)""" + class USSQualifierConfiguration(ImplicitDict): v1: Optional[USSQualifierConfigurationV1] diff --git a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml index 6ef5c7f17b..4bb83b7e9c 100644 --- a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml @@ -64,6 +64,18 @@ v1: reference_time: '2023-01-10T00:00:00.123456+00:00' time_start: '2023-01-10T00:00:01.123456+00:00' time_end: '2023-01-10T01:00:01.123456+00:00' + problematically_big_area: + resource_type: resources.VerticesResource + specification: + vertices: + - lat: 38 + lng: -81 + - lat: 37 + lng: -81 + - lat: 37 + lng: -80 + - lat: 38 + lng: -80 action: test_suite: suite_type: suites.interuss.dss.all_tests @@ -72,6 +84,7 @@ v1: f3411v22a_dss_instances: netrid_dss_instances_v22a id_generator: id_generator service_area: service_area + problematically_big_area: problematically_big_area artifacts: report: report_path: output/report_dss_probing.json @@ -86,3 +99,7 @@ v1: participant_requirements: uss1: all_astm_dss_requirements uss2: all_astm_dss_requirements + validation: + criteria: + - full_success: {} + - no_skipped_actions: {} diff --git a/monitoring/uss_qualifier/configurations/dev/f3548.yaml b/monitoring/uss_qualifier/configurations/dev/f3548.yaml index 507ac702cb..c06ccd1eff 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548.yaml @@ -28,3 +28,5 @@ v1: participant_requirements: uss1: scd uss2: scd + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml index 9be9d5dff6..368c203b0a 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml @@ -13,3 +13,5 @@ v1: artifacts: report: report_path: output/report_f3548_flight_intent_validation.json + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml index dcf680442c..5d38bcb43e 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml @@ -14,3 +14,5 @@ v1: artifacts: report: report_path: output/report_f3548_nominal_planning_conflict_equal_priority_not_permitted.json + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml index 67045d1553..846496d195 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml @@ -14,3 +14,5 @@ v1: artifacts: report: report_path: output/report_f3548_nominal_planning_conflict_higher_priority.json + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml index 68ceed7ce9..ee0dd0e734 100644 --- a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml +++ b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml @@ -12,4 +12,6 @@ v1: on_failure: Abort artifacts: report: - report_path: output/report_general_flight_auth.json \ No newline at end of file + report_path: output/report_general_flight_auth.json + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.json b/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.json index d890bdf71e..e06d3b0d09 100644 --- a/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.json +++ b/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.json @@ -85,6 +85,9 @@ } } } + }, + "validation": { + "$ref": "./library/validation.yaml#/normal_test" } } } diff --git a/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml b/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml index 283b520ed8..e44ba54145 100644 --- a/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml +++ b/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml @@ -17,3 +17,5 @@ v1: report_path: output/report_geoawareness_cis.json graph: gv_path: output/report_geoawareness_cis.gv + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml b/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml index bd9db1ce49..9e45824562 100644 --- a/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml +++ b/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml @@ -12,4 +12,6 @@ v1: on_failure: Abort artifacts: report: - report_path: output/report_geospatial_comprehension.json \ No newline at end of file + report_path: output/report_geospatial_comprehension.json + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/library/validation.yaml b/monitoring/uss_qualifier/configurations/dev/library/validation.yaml new file mode 100644 index 0000000000..b049c2247c --- /dev/null +++ b/monitoring/uss_qualifier/configurations/dev/library/validation.yaml @@ -0,0 +1,5 @@ +normal_test: + $content_schema: monitoring/uss_qualifier/reports/validation/report_validation/ValidationConfiguration.json + criteria: + - full_success: {} + - no_skipped_actions: {} diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml index 66169ae52f..9bc131053b 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml @@ -21,3 +21,5 @@ v1: report_path: output/report_netrid_v19.json tested_roles: report_path: output/tested_roles_netrid_v19 + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml index 21ae1bfc46..ed7c9ec0b1 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml @@ -37,3 +37,5 @@ v1: participant_requirements: uss1: sp_dp_dss uss2: sp_dss + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/noop.yaml b/monitoring/uss_qualifier/configurations/dev/noop.yaml index 21c93b7e62..95af477c6d 100644 --- a/monitoring/uss_qualifier/configurations/dev/noop.yaml +++ b/monitoring/uss_qualifier/configurations/dev/noop.yaml @@ -26,4 +26,6 @@ v1: artifacts: report: # Path to main report output - report_path: output/report_noop.json \ No newline at end of file + report_path: output/report_noop.json + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index b8f00eece7..51234d5769 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -89,3 +89,5 @@ v1: uss2: uspace sequence_view: output_path: output/sequence_uspace + validation: + $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py index a8e9afd2a7..8d1ab536f7 100644 --- a/monitoring/uss_qualifier/main.py +++ b/monitoring/uss_qualifier/main.py @@ -16,6 +16,7 @@ USSQualifierConfiguration, ArtifactsConfiguration, ReportConfiguration, + USSQualifierConfigurationV1, ) from monitoring.uss_qualifier.fileio import load_dict_with_references from monitoring.uss_qualifier.reports.documents import make_report_html @@ -27,6 +28,9 @@ from monitoring.uss_qualifier.reports.graphs import make_graph from monitoring.uss_qualifier.reports.report import TestRunReport, redact_access_tokens from monitoring.uss_qualifier.reports.templates import render_templates +from monitoring.uss_qualifier.reports.validation.report_validation import ( + validate_report, +) from monitoring.uss_qualifier.resources.resource import create_resources from monitoring.uss_qualifier.signatures import ( compute_signature, @@ -145,7 +149,7 @@ def main() -> int: logger.info("Exiting because --exit-before-execution specified.") return os.EX_OK - config = whole_config.v1 + config: USSQualifierConfigurationV1 = whole_config.v1 if args.report: if not config.artifacts: config.artifacts = ArtifactsConfiguration( @@ -210,6 +214,14 @@ def main() -> int: logger.info(f"Writing sequence view to {path}") generate_sequence_view(report, config.artifacts.sequence_view) + if "validation" in config and config.validation: + logger.info(f"Validating test run report for configuration '{args.config}'") + if not validate_report(report, config.validation): + logger.error( + f"Validation failed on test run report for configuration '{args.config}'" + ) + return -1 + return os.EX_OK diff --git a/monitoring/uss_qualifier/reports/validation/__init__.py b/monitoring/uss_qualifier/reports/validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/reports/validation/definitions.py b/monitoring/uss_qualifier/reports/validation/definitions.py new file mode 100644 index 0000000000..ab9f75bfe0 --- /dev/null +++ b/monitoring/uss_qualifier/reports/validation/definitions.py @@ -0,0 +1,28 @@ +from typing import Optional, List + +from implicitdict import ImplicitDict + + +class FullSuccessCriterion(ImplicitDict): + """Validation criterion that every element of the report must indicate success.""" + + pass + + +class NoSkippedActionsCriterion(ImplicitDict): + """Validation criterion that no actions in the entire test run may be skipped.""" + + pass + + +class ValidationCriterion(ImplicitDict): + """Wrapper for all the potential types of validation criteria.""" + + full_success: Optional[FullSuccessCriterion] = None + no_skipped_actions: Optional[NoSkippedActionsCriterion] = None + + +class ValidationConfiguration(ImplicitDict): + """Complete set of validation criteria that a test run report must satisfy.""" + + criteria: List[ValidationCriterion] diff --git a/monitoring/uss_qualifier/reports/validation/report_validation.py b/monitoring/uss_qualifier/reports/validation/report_validation.py new file mode 100644 index 0000000000..9f963dcd41 --- /dev/null +++ b/monitoring/uss_qualifier/reports/validation/report_validation.py @@ -0,0 +1,103 @@ +from loguru import logger + +from monitoring.monitorlib.dicts import JSONAddress +from monitoring.uss_qualifier.reports.report import TestRunReport, TestSuiteActionReport +from monitoring.uss_qualifier.reports.validation.definitions import ( + ValidationConfiguration, +) + + +def _validate_action_full_success( + report: TestSuiteActionReport, context: JSONAddress +) -> bool: + test_suite, test_scenario, action_generator = report.get_applicable_report() + if test_scenario: + success = report.test_scenario.successful + if not success: + logger.error( + f"Full success not achieved because {context}.test_scenario.successful was False" + ) + elif test_suite: + if report.test_suite.successful: + success = True + else: + success = False + logger.error( + f"Full success not achieved because {context}.test_suite.successful was False" + ) + for i, a in enumerate(report.test_suite.actions): + success = success and _validate_action_full_success( + a, JSONAddress(context + f".test_suite.actions[{i}]") + ) + elif action_generator: + if report.action_generator.successful: + success = True + else: + success = False + logger.error( + f"Full success not achieved because {context}.action_generator.successful was False" + ) + for i, a in enumerate(report.action_generator.actions): + success = success and _validate_action_full_success( + a, JSONAddress(context + f".action_generator.actions[{i}]") + ) + return success + + +def _validate_full_success(report: TestRunReport) -> bool: + return _validate_action_full_success(report.report, JSONAddress("$")) + + +def _validate_action_no_skipped_actions( + report: TestSuiteActionReport, context: JSONAddress +) -> bool: + test_suite, test_scenario, action_generator = report.get_applicable_report() + if test_scenario: + success = True + elif test_suite: + if ( + "skipped_actions" not in report.test_suite + or not report.test_suite.skipped_actions + ): + success = True + else: + success = False + for sa in report.test_suite.skipped_actions: + logger.error( + f"No skipped actions not achieved because {context}.test_suite had a skipped action for action index {sa.action_declaration_index}: {sa.reason}" + ) + for i, a in enumerate(report.test_suite.actions): + success = success and _validate_action_no_skipped_actions( + a, JSONAddress(context + f".test_suite.actions[{i}]") + ) + elif action_generator: + success = True + for i, a in enumerate(report.action_generator.actions): + success = success and _validate_action_no_skipped_actions( + a, JSONAddress(context + f".action_generator.actions[{i}]") + ) + return success + + +def _validate_no_skipped_actions(report: TestRunReport) -> bool: + return _validate_action_no_skipped_actions(report.report, JSONAddress("$")) + + +def validate_report(report: TestRunReport, validation: ValidationConfiguration) -> bool: + """Validate that the provided report meets all the specified validation criteria. + + Validation failures are logged as errors. + + Args: + report: Report to validate. + validation: Validation criteria. + + Returns: True if the report satisfies all criteria, False otherwise. + """ + success = True + for criterion in validation.criteria: + if criterion.full_success is not None: + success = success and _validate_full_success(report) + if criterion.no_skipped_actions is not None: + success = success and _validate_no_skipped_actions(report) + return success diff --git a/monitoring/uss_qualifier/run_locally.sh b/monitoring/uss_qualifier/run_locally.sh index 4555179d22..571e6d8220 100755 --- a/monitoring/uss_qualifier/run_locally.sh +++ b/monitoring/uss_qualifier/run_locally.sh @@ -60,7 +60,6 @@ else docker_args="-it" fi - start_time=$(date +%Y-%m-%dT%H:%M:%S) echo "========== Running uss_qualifier for configuration ${CONFIG_NAME} ==========" # shellcheck disable=SC2086 docker run ${docker_args} --name uss_qualifier \ @@ -77,18 +76,5 @@ else interuss/monitoring \ python main.py $QUALIFIER_OPTIONS echo "========== Completed uss_qualifier for configuration ${CONFIG_NAME} ==========" - - # Set return code according to whether the test run was fully successful - reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time") - # shellcheck disable=SC2068 - for REPORT in ${reports_generated[@]}; do - successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT") - if echo "${successful}" | grep -iqF true; then - echo "Full success indicated by $REPORT" - else - echo "Could not establish that all uss_qualifier tests passed in $REPORT" - exit 1 - fi - done fi diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.yaml b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.yaml index b8fad053be..e12522e2b6 100644 --- a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.yaml +++ b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.yaml @@ -4,6 +4,7 @@ resources: f3411v22a_dss_instances: resources.astm.f3411.DSSInstancesResource id_generator: resources.interuss.IDGeneratorResource service_area: resources.netrid.ServiceAreaResource + problematically_big_area: resources.VerticesResource actions: - action_generator: generator_type: action_generators.astm.f3411.ForEachDSS @@ -11,6 +12,7 @@ actions: dss_instances: f3411v19_dss_instances id_generator: id_generator service_area: service_area + problematically_big_area: problematically_big_area specification: action_to_repeat: test_suite: @@ -20,6 +22,7 @@ actions: all_dss_instances: dss_instances id_generator: id_generator isa: service_area + problematically_big_area: problematically_big_area on_failure: Continue dss_instances_source: dss_instances dss_instance_id: dss @@ -30,6 +33,7 @@ actions: dss_instances: f3411v22a_dss_instances id_generator: id_generator service_area: service_area + problematically_big_area: problematically_big_area specification: action_to_repeat: test_suite: @@ -39,6 +43,7 @@ actions: all_dss_instances: dss_instances id_generator: id_generator isa: service_area + problematically_big_area: problematically_big_area on_failure: Continue dss_instances_source: dss_instances dss_instance_id: dss diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/USSQualifierConfigurationV1.json b/schemas/monitoring/uss_qualifier/configurations/configuration/USSQualifierConfigurationV1.json index 44cb88d232..a45a3ddbed 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/USSQualifierConfigurationV1.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/USSQualifierConfigurationV1.json @@ -1,12 +1,25 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/USSQualifierConfigurationV1.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.USSQualifierConfigurationV1, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "artifacts": { + "description": "If specified, configuration describing the artifacts related to the test run", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "ArtifactsConfiguration.json" + } + ] }, "test_run": { + "description": "If specified, configuration describing how to perform a test run", "oneOf": [ { "type": "null" @@ -14,21 +27,19 @@ { "$ref": "TestConfiguration.json" } - ], - "description": "If specified, configuration describing how to perform a test run" + ] }, - "artifacts": { + "validation": { + "description": "If specified, configuration describing how to validate the output report (and return an error code if validation fails)", "oneOf": [ { "type": "null" }, { - "$ref": "ArtifactsConfiguration.json" + "$ref": "../../reports/validation/definitions/ValidationConfiguration.json" } - ], - "description": "If specified, configuration describing the artifacts related to the test run" + ] } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/USSQualifierConfigurationV1.json", - "description": "monitoring.uss_qualifier.configurations.configuration.USSQualifierConfigurationV1, as defined in monitoring/uss_qualifier/configurations/configuration.py" + "type": "object" } \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/validation/definitions/FullSuccessCriterion.json b/schemas/monitoring/uss_qualifier/reports/validation/definitions/FullSuccessCriterion.json new file mode 100644 index 0000000000..9644ce991a --- /dev/null +++ b/schemas/monitoring/uss_qualifier/reports/validation/definitions/FullSuccessCriterion.json @@ -0,0 +1,12 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/validation/definitions/FullSuccessCriterion.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Validation criterion that every element of the report must indicate success.\n\nmonitoring.uss_qualifier.reports.validation.definitions.FullSuccessCriterion, as defined in monitoring/uss_qualifier/reports/validation/definitions.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/validation/definitions/NoSkippedActionsCriterion.json b/schemas/monitoring/uss_qualifier/reports/validation/definitions/NoSkippedActionsCriterion.json new file mode 100644 index 0000000000..29ca95eff0 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/reports/validation/definitions/NoSkippedActionsCriterion.json @@ -0,0 +1,12 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/validation/definitions/NoSkippedActionsCriterion.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Validation criterion that no actions in the entire test run may be skipped.\n\nmonitoring.uss_qualifier.reports.validation.definitions.NoSkippedActionsCriterion, as defined in monitoring/uss_qualifier/reports/validation/definitions.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationConfiguration.json b/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationConfiguration.json new file mode 100644 index 0000000000..2d74bc1075 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationConfiguration.json @@ -0,0 +1,21 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Complete set of validation criteria that a test run report must satisfy.\n\nmonitoring.uss_qualifier.reports.validation.definitions.ValidationConfiguration, as defined in monitoring/uss_qualifier/reports/validation/definitions.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "criteria": { + "items": { + "$ref": "ValidationCriterion.json" + }, + "type": "array" + } + }, + "required": [ + "criteria" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationCriterion.json b/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationCriterion.json new file mode 100644 index 0000000000..86a503e3b4 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationCriterion.json @@ -0,0 +1,32 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/validation/definitions/ValidationCriterion.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Wrapper for all the potential types of validation criteria.\n\nmonitoring.uss_qualifier.reports.validation.definitions.ValidationCriterion, as defined in monitoring/uss_qualifier/reports/validation/definitions.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "full_success": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "FullSuccessCriterion.json" + } + ] + }, + "no_skipped_actions": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "NoSkippedActionsCriterion.json" + } + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/requirements/documentation/RequirementSet.json b/schemas/monitoring/uss_qualifier/requirements/documentation/RequirementSet.json deleted file mode 100644 index f733d2316d..0000000000 --- a/schemas/monitoring/uss_qualifier/requirements/documentation/RequirementSet.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" - }, - "requirement_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "type": "string" - } - }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/requirements/documentation/RequirementSet.json", - "description": "monitoring.uss_qualifier.requirements.documentation.RequirementSet, as defined in monitoring/uss_qualifier/requirements/documentation.py", - "required": [ - "name", - "requirement_ids" - ] -} \ No newline at end of file