diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index f60b1a8f4f..76c29fbdb6 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -29,6 +29,8 @@ UssAvailabilityState, UssAvailabilityStatusResponse, GetOperationalIntentReferenceResponse, + OPERATIONS, + OperationID, ) # A base URL for a USS that is not expected to be ever called @@ -77,12 +79,12 @@ def __init__( def find_op_intent( self, extent: Volume4D ) -> Tuple[List[OperationalIntentReference], fetch.Query]: - url = "/dss/v1/operational_intent_references/query" + op = OPERATIONS[OperationID.QueryOperationalIntentReferences] req = QueryOperationalIntentReferenceParameters(area_of_interest=extent) query = fetch.query_and_describe( self.client, - "POST", - url, + op.verb, + op.path, QueryType.F3548v21DSSQueryOperationalIntentReferences, self.participant_id, scope=SCOPE_SC, @@ -103,11 +105,11 @@ def get_op_intent_reference( """ Retrieve an OP Intent from the DSS, using only its ID """ - url = f"/dss/v1/operational_intent_references/{op_intent_id}" + op = OPERATIONS[OperationID.GetOperationalIntentReference] query = fetch.query_and_describe( self.client, - "GET", - url, + op.verb, + op.path.format(entityid=op_intent_id), QueryType.F3548v21DSSGetOperationalIntentReference, self.participant_id, scope=SCOPE_SC, @@ -125,11 +127,11 @@ def get_full_op_intent( op_intent_ref: OperationalIntentReference, uss_participant_id: Optional[str] = None, ) -> Tuple[OperationalIntent, fetch.Query]: - url = f"{op_intent_ref.uss_base_url}/uss/v1/operational_intents/{op_intent_ref.id}" + op = OPERATIONS[OperationID.GetOperationalIntentDetails] query = fetch.query_and_describe( self.client, - "GET", - url, + op.verb, + f"{op_intent_ref.uss_base_url}{op.path.format(entityid=op_intent_ref.id)}", QueryType.F3548v21USSGetOperationalIntentDetails, uss_participant_id, scope=SCOPE_SC, @@ -157,10 +159,12 @@ def put_op_intent( ]: oi_uuid = str(uuid.uuid4()) if id is None else id if ovn is None: - url = f"/dss/v1/operational_intent_references/{oi_uuid}" + op = OPERATIONS[OperationID.CreateOperationalIntentReference] + url = op.path.format(entityid=oi_uuid) query_type = QueryType.F3548v21DSSCreateOperationalIntentReference else: - url = f"/dss/v1/operational_intent_references/{oi_uuid}/{ovn}" + op = OPERATIONS[OperationID.UpdateOperationalIntentReference] + url = op.path.format(entityid=oi_uuid, ovn=ovn) query_type = QueryType.F3548v21DSSUpdateOperationalIntentReference req = PutOperationalIntentReferenceParameters( @@ -172,7 +176,7 @@ def put_op_intent( ) query = fetch.query_and_describe( self.client, - "PUT", + op.verb, url, query_type, self.participant_id, @@ -198,10 +202,11 @@ def delete_op_intent( Optional[List[SubscriberToNotify]], fetch.Query, ]: + op = OPERATIONS[OperationID.DeleteOperationalIntentReference] query = fetch.query_and_describe( self.client, - "DELETE", - f"/dss/v1/operational_intent_references/{id}/{ovn}", + op.verb, + op.path.format(entityid=id, ovn=ovn), QueryType.F3548v21DSSDeleteOperationalIntentReference, self.participant_id, scope=SCOPE_SC, @@ -237,10 +242,11 @@ def set_uss_availability( old_version=version, availability=availability, ) + op = OPERATIONS[OperationID.SetUssAvailability] query = fetch.query_and_describe( self.client, - "PUT", - f"/dss/v1/uss_availability/{uss_id}", + op.verb, + op.path.format(uss_id=uss_id), QueryType.F3548v21DSSSetUssAvailability, self.participant_id, scope=SCOPE_AA, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md index 959526a49e..5ea938f54b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md @@ -20,3 +20,17 @@ planners provided as resource are used to determine and evaluate the 95th percen If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second), this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**. + +## Interoperability test instance is available test case + +### Interoperability test instance is available test step + +This step verifies that interactions with the interoperability test instances happened and where at least partly successful. + +#### Interoperability test instance is available check + +This check ensures that interactions with the interoperability test instance that each USS must provide are possible. + +If all interactions fail, or if no test instance can be reached, the USS is failing to meet **[astm.f3548.v21.GEN0300](../../../requirements/astm/f3548/v21.md)**. + +If no interaction with a test instance was found, this check is skipped. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py index bd4dc50afd..79657c3d36 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py @@ -1,19 +1,17 @@ from typing import List, Dict +from uas_standards.astm.f3548.v21 import constants + from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import evaluation, QueryType from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ParticipantID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource -from monitoring.uss_qualifier.suites.suite import ExecutionContext - -from uas_standards.astm.f3548.v21 import constants - from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.suites.suite import ExecutionContext class AggregateChecks(TestScenario): - _queries: List[fetch.Query] _attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {} @@ -76,6 +74,14 @@ def run(self, context: ExecutionContext): self.end_test_step() self.end_test_case() + self.begin_test_case("Interoperability test instance is available") + self.begin_test_step("Interoperability test instance is available") + + self._confirm_test_harness_queries_work() + + self.end_test_step() + self.end_test_case() + self.end_test_scenario() def _op_intent_details_step(self): @@ -118,3 +124,66 @@ def _op_intent_details_step(self): f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}", f"checked performances on {len(durations)} queries, 95th percentile: {p95}s", ) + + def _confirm_test_harness_queries_work(self): + """ + For each different type of call to the interoperability test instance, + we look for at least one successful query. + """ + for participant, queries_by_type in self._attributed_queries.items(): + self._validate_participant_test_interop_instance( + participant, queries_by_type + ) + + def _validate_participant_test_interop_instance( + self, + participant_id: str, + participant_queries: dict[QueryType, List[fetch.Query]], + ): + # Keep track of how many interactions we've found for this participant + # if there is None the condition is not met + test_interactions = 0 + success_by_type: Dict[QueryType, bool] = {} + for query_type, queries in participant_queries.items(): + if _is_interop_test_interaction(query_type): + test_interactions += len(queries) + success_by_type[query_type] = False + for query in queries: + if 200 <= query.response.status_code < 300: + success_by_type[query_type] = True + break + + self.record_note( + "test_interop_interactions", + f"Found {test_interactions} interactions with interoperability test instance for {participant_id}", + ) + if test_interactions == 0: + self.record_note( + "test_interop_check_skipped", + f"Skipping check for {participant_id} because no interactions with " + f"interoperability test instance were found", + ) + # If no interactions are observed, we can't determine if the test instance is available + # and the step here. + return + + with self.check( + "Interoperability test instance is available", [participant_id] + ) as check: + for query_type, success in success_by_type.items(): + if not success: + check.record_failed( + summary=f"No successful {query_type} interaction with interoperability test instance", + severity=Severity.Medium, + details=f"Found no successful {query_type} interaction with interoperability test instance, " + f"indicating that the test instance is either not available or not properly implemented.", + ) + + +def _is_interop_test_interaction(query_type: QueryType): + return ( + query_type == QueryType.InterUSSFlightPlanningV1GetStatus + or query_type == QueryType.InterUSSFlightPlanningV1ClearArea + or query_type == QueryType.InterUSSFlightPlanningV1UpsertFlightPlan + or query_type == QueryType.InterUSSFlightPlanningV1DeleteFlightPlan + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md index fec8fa805a..5acf0a836a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md @@ -211,6 +211,7 @@ Because the modification attempt was invalid, either Flight 1 should not have be original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). + ## Cleanup ### Successful flight deletion check **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 3b22fdfe11..4dd60ecf53 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -31,7 +31,7 @@ Checked in - astm
.f3548
.v21
+ astm
.f3548
.v21
DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -46,6 +46,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 5eb1335141..972a811bd8 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21
+ astm
.f3548
.v21
DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -33,6 +33,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 8256df3cd4..a8956efd0e 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -19,7 +19,7 @@ Checked in - astm
.f3548
.v21
+ astm
.f3548
.v21
DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -34,6 +34,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index ccc007b6f5..e96c85f8f7 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -449,7 +449,7 @@ ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation
ASTM NetRID DSS: Token Validation - astm
.f3548
.v21
+ astm
.f3548
.v21
DSS0005 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Access Control
Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted
Validation of operational intents @@ -464,6 +464,11 @@ Implemented ASTM F3548-21 UTM DSS interoperability + + GEN0300 + Implemented + ASTM F3548 UTM aggregate checks + GEN0310 Implemented