From 2230d7b56eb407fcad2d4ce6e2f47ef4d0902a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Wed, 27 Dec 2023 00:15:05 +0100 Subject: [PATCH] [uss_qualifier/scenarios/utm] Add requirement SCD0100 check; Fix some validation issues (#444) * [uss_qualifier/scenarios/utm] Add requirement SCD0100 check; Fix some validation issues * add mock_uss endpoint for telemetry --- monitoring/mock_uss/f3548v21/routes_scd.py | 58 +++- .../resources/astm/f3548/v21/dss.py | 27 +- .../scenarios/astm/utm/test_steps.py | 279 ++++++++++-------- .../utm/validate_shared_operational_intent.md | 6 + .../uss_qualifier/suites/astm/utm/f3548_21.md | 7 +- .../suites/faa/uft/message_signing.md | 7 +- .../suites/uspace/flight_auth.md | 7 +- .../suites/uspace/required_services.md | 7 +- 8 files changed, 261 insertions(+), 137 deletions(-) diff --git a/monitoring/mock_uss/f3548v21/routes_scd.py b/monitoring/mock_uss/f3548v21/routes_scd.py index 0bb78bc04d..e8bc6b336a 100644 --- a/monitoring/mock_uss/f3548v21/routes_scd.py +++ b/monitoring/mock_uss/f3548v21/routes_scd.py @@ -1,13 +1,17 @@ +from typing import Optional + import flask from monitoring.mock_uss.f3548v21.flight_planning import op_intent_from_flightrecord from monitoring.monitorlib import scd from monitoring.mock_uss import webapp from monitoring.mock_uss.auth import requires_scope -from monitoring.mock_uss.flights.database import db +from monitoring.mock_uss.flights.database import db, FlightRecord from uas_standards.astm.f3548.v21.api import ( ErrorResponse, GetOperationalIntentDetailsResponse, + GetOperationalIntentTelemetryResponse, + OperationalIntentState, ) @@ -44,6 +48,58 @@ def scdsc_get_operational_intent_details(entityid: str): return flask.jsonify(response), 200 +@webapp.route( + "/mock/scd/uss/v1/operational_intents//telemetry", methods=["GET"] +) +@requires_scope(scd.SCOPE_CM_SA) +def scdsc_get_operational_intent_telemetry(entityid: str): + """Implements getOperationalIntentTelemetry in ASTM SCD API.""" + + # Look up entityid in database + tx = db.value + flight: Optional[FlightRecord] = None + for f in tx.flights.values(): + if f and f.op_intent.reference.id == entityid: + flight = f + break + + # If requested operational intent doesn't exist, return 404 + if flight is None: + return ( + flask.jsonify( + ErrorResponse( + message="Operational intent {} not known by this USS".format( + entityid + ) + ) + ), + 404, + ) + + elif flight.op_intent.reference.state not in { + OperationalIntentState.Contingent, + OperationalIntentState.Nonconforming, + }: + return ( + flask.jsonify( + ErrorResponse( + message=f"Operational intent {entityid} is not in a state that provides telemetry ({flight.op_intent.reference.state})" + ) + ), + 409, + ) + + # TODO: implement support for telemetry + return ( + flask.jsonify( + ErrorResponse( + message=f"Operational intent {entityid} has no telemetry data available." + ) + ), + 412, + ) + + @webapp.route("/mock/scd/uss/v1/operational_intents", methods=["POST"]) @requires_scope(scd.SCOPE_SC) def scdsc_notify_operational_intent_details_changed(): diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index aabb03bc50..f9889b6d12 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -7,7 +7,7 @@ from monitoring.monitorlib import infrastructure, fetch from monitoring.monitorlib.fetch import QueryType -from monitoring.monitorlib.scd import SCOPE_SC, SCOPE_AA +from monitoring.monitorlib.scd import SCOPE_SC, SCOPE_AA, SCOPE_CM_SA from monitoring.uss_qualifier.resources.resource import Resource from monitoring.uss_qualifier.resources.communications import AuthAdapterResource from uas_standards.astm.f3548.v21.api import ( @@ -30,6 +30,8 @@ GetOperationalIntentReferenceResponse, OPERATIONS, OperationID, + GetOperationalIntentTelemetryResponse, + VehicleTelemetry, ) # A base URL for a USS that is not expected to be ever called @@ -164,6 +166,29 @@ def get_full_op_intent_without_validation( return result, query + def get_op_intent_telemetry( + self, + op_intent_ref: OperationalIntentReference, + uss_participant_id: Optional[str] = None, + ) -> Tuple[Optional[VehicleTelemetry], fetch.Query]: + op = OPERATIONS[OperationID.GetOperationalIntentTelemetry] + query = fetch.query_and_describe( + self.client, + op.verb, + f"{op_intent_ref.uss_base_url}{op.path.format(entityid=op_intent_ref.id)}", + QueryType.F3548v21USSGetOperationalIntentTelemetry, + uss_participant_id, + scope=SCOPE_CM_SA, + ) + if query.status_code == 200: + result: GetOperationalIntentTelemetryResponse = ImplicitDict.parse( + query.response.json, GetOperationalIntentTelemetryResponse + ) + telemetry = result.telemetry if "telemetry" in result else None + return telemetry, query + else: + return None, query + def put_op_intent( self, extents: List[Volume4D], diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index bde5bb26da..e9549c930b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -5,7 +5,6 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import schema_validation, fetch from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient -from monitoring.monitorlib.geotemporal import Volume4DCollection from uas_standards.astm.f3548.v21.api import ( OperationalIntentState, Volume4D, @@ -31,11 +30,6 @@ ) from uas_standards.interuss.automated_testing.scd.v1.api import InjectFlightRequest -OI_DATA_FORMAT = "Operational intent details data format" -OI_CORRECT_DETAILS = "Correct operational intent details" -OFF_NOM_VOLS = "Off-nominal volumes" -VERTICES = "Vertices" - class OpIntentValidator(object): """ @@ -165,109 +159,26 @@ def expect_shared( :returns: the shared operational intent reference. None if skipped because not found. """ - oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) - - oi_full, oi_full_query = self._dss.get_full_op_intent( - oi_ref, self._flight_planner.participant_id - ) - self._scenario.record_query(oi_full_query) - self._operational_intent_retrievable_check(oi_full_query, oi_ref.id) - - validation_failures = self._evaluate_op_intent_validation(oi_full_query) + if isinstance(flight_intent, InjectFlightRequest): + flight_intent = FlightInfo.from_scd_inject_flight_request(flight_intent) - with self._scenario.check( - OI_DATA_FORMAT, - [self._flight_planner.participant_id], - ) as check: - data_format_fail = ( - self._expected_validation_failure_found( - validation_failures, OpIntentValidationFailureType.DataFormat - ) - if validation_failures - else None - ) - if data_format_fail: - errors = data_format_fail.errors - check.record_failed( - summary="Operational intent details response failed schema validation", - severity=Severity.Medium, - details="The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" - + "\n".join( - f"At {e.json_path} in the response: {e.message}" for e in errors - ), - query_timestamps=[oi_full_query.request.timestamp], - ) - - with self._scenario.check( - OI_CORRECT_DETAILS, [self._flight_planner.participant_id] - ) as check: - priority = ( - flight_intent.operational_intent.priority - if isinstance(flight_intent, InjectFlightRequest) - else flight_intent.astm_f3548_21.priority - ) - if isinstance(flight_intent, InjectFlightRequest): - priority = flight_intent.operational_intent.priority - vols = Volume4DCollection.from_interuss_scd_api( - flight_intent.operational_intent.volumes - + flight_intent.operational_intent.off_nominal_volumes - ) - elif isinstance(flight_intent, FlightInfo): - priority = flight_intent.astm_f3548_21.priority - vols = flight_intent.basic_information.area - - error_text = validate_op_intent_details( - oi_full.details, - priority, - vols.bounding_volume.to_f3548v21(), - ) - if error_text: - check.record_failed( - summary="Operational intent details do not match user flight intent", - severity=Severity.High, - details=error_text, - query_timestamps=[oi_full_query.request.timestamp], - ) + self._begin_step() + oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) + if oi_ref is None: + self._scenario.end_test_step() + return None - with self._scenario.check( - OFF_NOM_VOLS, [self._flight_planner.participant_id] - ) as check: - off_nom_vol_fail = ( - self._expected_validation_failure_found( - validation_failures, - OpIntentValidationFailureType.NominalWithOffNominalVolumes, - ) - if validation_failures - else None - ) - if off_nom_vol_fail: - check.record_failed( - summary="Accepted or Activated operational intents are not allowed off-nominal volumes", - severity=Severity.Medium, - details=off_nom_vol_fail.error_text, - query_timestamps=[oi_full_query.request.timestamp], - ) + self._check_op_intent_details(flight_intent, oi_ref) - with self._scenario.check( - VERTICES, [self._flight_planner.participant_id] - ) as check: - vertices_fail = ( - self._expected_validation_failure_found( - validation_failures, OpIntentValidationFailureType.VertexCount - ) - if validation_failures - else None - ) - if vertices_fail: - check.record_failed( - summary="Too many vertices", - severity=Severity.Medium, - details=vertices_fail.error_text, - query_timestamps=[oi_full_query.request.timestamp], - ) + # Check telemetry if intent is off-nominal + if flight_intent.basic_information.uas_state in { + UasState.OffNominal, + UasState.Contingent, + }: + self._check_op_intent_telemetry(oi_ref) self._scenario.end_test_step() - return oi_full.reference + return oi_ref def expect_shared_with_invalid_data( self, @@ -287,8 +198,14 @@ def expect_shared_with_invalid_data( :returns: the shared operational intent reference. None if skipped because not found. """ + if isinstance(flight_intent, InjectFlightRequest): + flight_intent = FlightInfo.from_scd_inject_flight_request(flight_intent) + self._begin_step() oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) + if oi_ref is None: + self._scenario.end_test_step() + return None goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( oi_ref, self._flight_planner.participant_id @@ -335,11 +252,9 @@ def _operational_intent_retrievable_check( def _operational_intent_shared_check( self, - flight_intent: Union[InjectFlightRequest | FlightInfo], + flight_intent: FlightInfo, skip_if_not_found: bool, - ) -> OperationalIntentReference: - - self._begin_step() + ) -> Optional[OperationalIntentReference]: with self._scenario.check( "Operational intent shared correctly", [self._flight_planner.participant_id] @@ -360,7 +275,6 @@ def _operational_intent_shared_check( f"{self._flight_planner.participant_id} skipped step", f"No new operational intent was found in DSS, instructed to skip test step '{self._test_step}'.", ) - self._scenario.end_test_step() return None oi_ref = self._new_oi_ref @@ -373,23 +287,11 @@ def _operational_intent_shared_check( if modified_oi_ref is None: if not skip_if_not_found: if ( - (isinstance(flight_intent, InjectFlightRequest)) - and ( - flight_intent.operational_intent.state - == OperationalIntentState.Activated - ) - ) or ( - isinstance(flight_intent, FlightInfo) - and ( - ( - flight_intent.basic_information.uas_state - == UasState.Nominal - ) - and ( - flight_intent.basic_information.usage_state - == AirspaceUsageState.InUse - ) - ) + flight_intent.basic_information.uas_state + == UasState.Nominal + ) and ( + flight_intent.basic_information.usage_state + == AirspaceUsageState.InUse ): with self._scenario.check( "Operational intent for active flight not deleted", @@ -415,7 +317,6 @@ def _operational_intent_shared_check( f"{self._flight_planner.participant_id} skipped step", f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS, instructed to skip test step '{self._test_step}'.", ) - self._scenario.end_test_step() return None oi_ref = modified_oi_ref @@ -432,6 +333,118 @@ def _operational_intent_shared_check( return oi_ref + def _check_op_intent_details( + self, flight_intent: FlightInfo, oi_ref: OperationalIntentReference + ): + oi_full, oi_full_query = self._dss.get_full_op_intent( + oi_ref, self._flight_planner.participant_id + ) + self._scenario.record_query(oi_full_query) + self._operational_intent_retrievable_check(oi_full_query, oi_ref.id) + + validation_failures = self._evaluate_op_intent_validation(oi_full_query) + with self._scenario.check( + "Operational intent details data format", + [self._flight_planner.participant_id], + ) as check: + data_format_fail = ( + self._expected_validation_failure_found( + validation_failures, OpIntentValidationFailureType.DataFormat + ) + if validation_failures + else None + ) + if data_format_fail: + errors = data_format_fail.errors + check.record_failed( + summary="Operational intent details response failed schema validation", + severity=Severity.Medium, + details="The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" + + "\n".join( + f"At {e.json_path} in the response: {e.message}" for e in errors + ), + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + "Correct operational intent details", [self._flight_planner.participant_id] + ) as check: + error_text = validate_op_intent_details( + oi_full.details, + flight_intent.astm_f3548_21.priority, + flight_intent.basic_information.area.bounding_volume.to_f3548v21(), + ) + if error_text: + check.record_failed( + summary="Operational intent details do not match user flight intent", + severity=Severity.High, + details=error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + "Off-nominal volumes", [self._flight_planner.participant_id] + ) as check: + off_nom_vol_fail = ( + self._expected_validation_failure_found( + validation_failures, + OpIntentValidationFailureType.NominalWithOffNominalVolumes, + ) + if validation_failures + else None + ) + if off_nom_vol_fail: + check.record_failed( + summary="Accepted or Activated operational intents are not allowed off-nominal volumes", + severity=Severity.Medium, + details=off_nom_vol_fail.error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + "Vertices", [self._flight_planner.participant_id] + ) as check: + vertices_fail = ( + self._expected_validation_failure_found( + validation_failures, OpIntentValidationFailureType.VertexCount + ) + if validation_failures + else None + ) + if vertices_fail: + check.record_failed( + summary="Too many vertices", + severity=Severity.Medium, + details=vertices_fail.error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + def _check_op_intent_telemetry(self, oi_ref: OperationalIntentReference): + oi_tel, oi_tel_query = self._dss.get_op_intent_telemetry( + oi_ref, self._flight_planner.participant_id + ) + self._scenario.record_query(oi_tel_query) + + with self._scenario.check( + "Operational intent telemetry retrievable", + [self._flight_planner.participant_id], + ) as check: + if oi_tel_query.status_code not in {200, 412}: + check.record_failed( + summary="Operational intent telemetry could not be retrieved from USS", + severity=Severity.High, + details=f"Received status code {oi_tel_query.status_code} from {self._flight_planner.participant_id} when querying for telemetry of operational intent {oi_ref.id}", + query_timestamps=[oi_tel_query.request.timestamp], + ) + + if oi_tel is None: + check.record_failed( + summary="Warning (not a failure): USS indicated that no operational intent telemetry was available", + severity=Severity.Low, + details=f"Received status code {oi_tel_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {oi_ref.id}", + query_timestamps=[oi_tel_query.request.timestamp], + ) + def _evaluate_op_intent_validation( self, oi_full_query: fetch.Query ) -> Set[OpIntentValidationFailure]: @@ -485,13 +498,17 @@ def volume_vertices(v4): f"Operational intent {oi_full.reference.id} had too many total vertices - {n_vertices}", ) validation_failures.add( - validation_failure_type=OpIntentValidationFailureType.VertexCount, - error_text=details, + OpIntentValidationFailure( + validation_failure_type=OpIntentValidationFailureType.VertexCount, + error_text=details, + ) ) except (KeyError, ValueError) as e: validation_failures.add( - validation_failure_type=OpIntentValidationFailureType.DataFormat, - error_text=e, + OpIntentValidationFailure( + validation_failure_type=OpIntentValidationFailureType.DataFormat, + error_text=e, + ) ) return validation_failures diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md index cf89e31a83..694a381462 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md @@ -36,3 +36,9 @@ If the operational intent details reported by the USS do not match the user's fl ## ⚠️ Vertices check **[astm.f3548.v21.OPIN0020](../../../requirements/astm/f3548/v21.md)** + +## 🛑 Operational intent telemetry retrievable check + +If the operational intent is in an off-nominal state and that its telemetry cannot be retrieved from the USS, this check will fail per **[astm.f3548.v21.SCD0100](../../../requirements/astm/f3548/v21.md)**. + +The USS may explicitly indicate that no telemetry is available for this operational intent, in which case, as a warning, this check will fail with a low severity. diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 6b63849983..8fe439bb04 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -31,7 +31,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -156,6 +156,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 3ff13eb844..7dfe2bd883 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -143,6 +143,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index e94aec6801..5b924a7e21 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -19,7 +19,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -144,6 +144,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index b1399f5b71..6289673632 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -454,7 +454,7 @@ ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation
ASTM NetRID DSS: Token Validation - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -579,6 +579,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented