diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index a066f622ee..69247f0396 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -127,6 +127,8 @@ v1: - astm.f3411.v22a.display_provider#Operator Position transmitter - astm.f3411.v22a.dss_provider - astm.f3548.v21.scd#Automated verification + - requirements: + - uspace.article8.MSLAltitude participant_requirements: uss1: uspace uss2: uspace diff --git a/monitoring/uss_qualifier/requirements/uspace/article8.md b/monitoring/uss_qualifier/requirements/uspace/article8.md new file mode 100644 index 0000000000..ed8deb9c93 --- /dev/null +++ b/monitoring/uss_qualifier/requirements/uspace/article8.md @@ -0,0 +1,21 @@ +# [U-space Article 8](https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:32021R0664&qid=1702917443967#d1e905-161-1): Network Identification Service + +## Article 8(2)(c) + +> The network identification service shall allow for the authorised users to receive messages with the geographical position of the UAS, its altitude above mean sea level and its height above the surface or take-off point. + +MSLAltitude: Authorised users must receive messages with UAS altitude above mean sea level. + +## Article 8(4) + +> The authorised users shall be: +> +> (a) the general public as regards information that is deemed public in accordance with applicable Union and national rules; +> +> (b) other U-space service providers in order to ensure the safety of operations in the U-space airspace; +> +> (c) the air traffic services providers concerned; +> +> (d) when designated, the single common information service provider; +> +> (e) the relevant competent authorities. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index 754ed328ba..6e7a6e2abc 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -188,7 +188,6 @@ def _inspect_participant_queries( summary=f"found {len(cleartext_queries)} cleartext http queries", details=f"unique cleartext urls: {urls}", severity=Severity.Medium, - participants=[participant_id], query_timestamps=timestamps, ) else: @@ -233,14 +232,12 @@ def _dp_display_data_details_times_step(self): check.record_failed( summary=f"95th percentile of durations for DP display_data details queries is higher than threshold", severity=Severity.Medium, - participants=[participant], details=f"threshold: {self._rid_version.dp_details_resp_percentile95_s}s, 95th percentile: {p95}s", ) if p99 > self._rid_version.dp_details_resp_percentile99_s: check.record_failed( summary=f"99th percentile of durations for DP display_data details queries is higher than threshold", severity=Severity.Medium, - participants=[participant], details=f"threshold: {self._rid_version.dp_details_resp_percentile99_s}s, 99th percentile: {p99}s", ) @@ -279,7 +276,6 @@ def _sp_flights_area_times_step(self): check.record_failed( summary=f"95th percentile of /flights?view requests is {p95} s", severity=Severity.Medium, - participants=[participant], details=f"expected less than {self._rid_version.sp_data_resp_percentile95_s} s, was {p95}", ) with self.check("99th percentile response time", [participant]) as check: @@ -287,7 +283,6 @@ def _sp_flights_area_times_step(self): check.record_failed( summary=f"99th percentile of /flights?view requests is {p99} s", severity=Severity.Medium, - participants=[participant], details=f"expected less than {self._rid_version.sp_data_resp_percentile99_s} s, was {p99}", ) @@ -339,14 +334,12 @@ def _dp_display_data_times_step(self): check.record_failed( summary=f"95th percentile of durations for initial DP display_data queries is higher than threshold", severity=Severity.Medium, - participants=[participant], details=f"threshold: {self._rid_version.dp_init_resp_percentile95_s}, 95th percentile: {init_95th}", ) if init_99th > self._rid_version.dp_init_resp_percentile99_s: check.record_failed( summary=f"99th percentile of durations for initial DP display_data queries is higher than threshold", severity=Severity.Medium, - participants=[participant], details=f"threshold: {self._rid_version.dp_init_resp_percentile99_s}, 99th percentile: {init_99th}", ) @@ -357,14 +350,12 @@ def _dp_display_data_times_step(self): check.record_failed( summary=f"95th percentile of durations for subsequent DP display_data queries is higher than threshold", severity=Severity.Medium, - participants=[participant], details=f"threshold: {self._rid_version.dp_data_resp_percentile95_s}, 95th percentile: {subsequent_95th}", ) if subsequent_99th > self._rid_version.dp_data_resp_percentile99_s: check.record_failed( summary=f"99th percentile of durations for subsequent DP display_data queries is higher than threshold", severity=Severity.Medium, - participants=[participant], details=f"threshold: {self._rid_version.dp_data_resp_percentile99_s}, 95th percentile: {subsequent_99th}", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py index 0487202ff9..1007381a3e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py @@ -98,7 +98,6 @@ def _check_expiry_behaviors(self): check.record_failed( summary=f"Expired ISA {self._isa_id} found in search results", severity=Severity.Medium, - participants=[self._dss.participant_id], details=f"Searched for area {self._isa_area} with unspecified end and start time.", query_timestamps=[ created_isa.dss_query.query.request.timestamp, diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py index f93923ee1d..d5a73fe812 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py @@ -136,7 +136,6 @@ def _new_subscription_in_isa_step(self): check.record_failed( summary="Subscription response does not include the freshly created ISA", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to contain the ISA created for this same area. The returned subscription did not mention it.", query_timestamps=[ created_isa.dss_query.query.request.timestamp, @@ -152,7 +151,6 @@ def _new_subscription_in_isa_step(self): check.record_failed( summary="Subscription notification_index is not 0", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 0. The returned subscription has a notification_index of {created_subscription.subscription.notification_index}.", query_timestamps=[created_subscription.query.request.timestamp], ) @@ -188,7 +186,6 @@ def _new_subscription_in_isa_step(self): check.record_failed( summary="ISA mutation response does not contain expected subscription ID", severity=Severity.High, - participants=[self._dss.participant_id], details="Mutating an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.", query_timestamps=[ created_isa.dss_query.query.request.timestamp, @@ -210,7 +207,6 @@ def _new_subscription_in_isa_step(self): check.record_failed( summary="Subscription notification_index has not been increased", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 1 or more. The returned subscription has a notification_index of {subs_to_mutated_isa[created_subscription.subscription.id].notification_index}.", query_timestamps=[created_subscription.query.request.timestamp], ) @@ -244,7 +240,6 @@ def _new_subscription_in_isa_step(self): check.record_failed( summary="ISA deletion response does not contain expected subscription ID", severity=Severity.High, - participants=[self._dss.participant_id], details="Deleting an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.", query_timestamps=[ created_isa.dss_query.query.request.timestamp, @@ -285,7 +280,6 @@ def _new_subscription_in_isa_step(self): check.record_failed( summary="Subscription notification_index has not been incremented", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to have its notification increased after the subscription was deleted." f"The returned subscription has a notification_index of {subs_after_deletion.notification_index}, whilte the previous notification_index for that subscription was {sub_to_mutated_isa.notification_index}", query_timestamps=[created_subscription.query.request.timestamp], @@ -353,7 +347,6 @@ def _mutate_subscription_towards_isa_boundary_step(self): check.record_failed( summary="Subscription response does not include the freshly created ISA", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to contain the ISA created for this same area. The returned subscription did not mention it.", query_timestamps=[ created_isa.dss_query.query.request.timestamp, @@ -369,7 +362,6 @@ def _mutate_subscription_towards_isa_boundary_step(self): check.record_failed( summary="Subscription notification_index is not 0", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 0. The returned subscription has a notification_index of {created_subscription.subscription.notification_index}.", query_timestamps=[created_subscription.query.request.timestamp], ) @@ -405,7 +397,6 @@ def _mutate_subscription_towards_isa_boundary_step(self): check.record_failed( summary="ISA mutation response does not contain expected subscription ID", severity=Severity.High, - participants=[self._dss.participant_id], details="Mutating an ISA to which a subscription was made and then subsequently moved to the ISA's boundary," " the DSS failed to return the subscription ID in the response.", query_timestamps=[ @@ -429,7 +420,6 @@ def _mutate_subscription_towards_isa_boundary_step(self): check.record_failed( summary="Subscription notification_index has not been increased", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 1 or more. The returned subscription has a notification_index of {subs_to_mutated_isa[created_subscription.subscription.id].notification_index}.", query_timestamps=[created_subscription.query.request.timestamp], ) @@ -463,7 +453,6 @@ def _mutate_subscription_towards_isa_boundary_step(self): check.record_failed( summary="ISA deletion response does not contain expected subscription ID", severity=Severity.High, - participants=[self._dss.participant_id], details="Deleting an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.", query_timestamps=[ created_isa.dss_query.query.request.timestamp, @@ -504,7 +493,6 @@ def _mutate_subscription_towards_isa_boundary_step(self): check.record_failed( summary="Subscription notification_index has not been incremented", severity=Severity.High, - participants=[self._dss.participant_id], details=f"The subscription created for the area {self._isa_area} is expected to have its notification increased after the subscription was deleted." f"The returned subscription has a notification_index of {subs_after_deletion.notification_index}, whilte the previous notification_index for that subscription was {sub_to_mutated_isa.notification_index}", query_timestamps=[created_subscription.query.request.timestamp], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py index 376ca44586..93573d0a5e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py @@ -130,7 +130,6 @@ def _test_env_reqs(self): check.record_failed( summary=f"DSS host {parsed_url.netloc} is not publicly addressable", severity=Severity.Medium, - participants=[dss.participant_id], details=f"DSS (URL: {dss.base_url}, netloc: {parsed_url.netloc}, resolved IP: {ip_addr}) is not publicly addressable", ) @@ -184,7 +183,6 @@ def step2(self): check.record_failed( summary=f"DSS did not return ISA {isa_1.uuid} from testStep1 when creating Subscription {sub_1.uuid}", severity=Severity.High, - participants=[dss.participant_id], details=f"service_areas IDs: {', '.join([isa.id for isa in created_sub.isas])}", query_timestamps=[created_sub.query.request.timestamp], ) @@ -206,7 +204,6 @@ def get_fail_params( summary=f"ISA[{dss.participant_id}].{field_name} not equal ISA[{self._dss_primary.participant_id}].{field_name}", details=f"ISA[{dss.participant_id}].{field_name} is {primary_isa_field_value}; ISA[{self._dss_primary.participant_id}].{field_name} is {other_isa_field_value}", severity=Severity.High, - participants=[dss.participant_id], query_timestamps=[created_sub.query.request.timestamp], ) @@ -304,7 +301,6 @@ def get_fail_params( summary=f"Subscription[{dss.participant_id}].{field_name} not equal Subscription[{self._dss_primary.participant_id}].{field_name}", details=f"Subscription[{dss.participant_id}].{field_name} is {primary_sub_field_value}; Subscription[{self._dss_primary.participant_id}].{field_name} is {other_sub_field_value}", severity=Severity.High, - participants=[dss.participant_id], query_timestamps=[other_sub.query.request.timestamp], ) @@ -417,7 +413,6 @@ def step4(self): check.record_failed( summary=f"DSS returned too few subscriptions", severity=Severity.High, - participants=[dss.participant_id], details=f"Missing: {', '.join(missing_subs)}", query_timestamps=[subs.query.request.timestamp], ) @@ -496,7 +491,6 @@ def step8(self): check.record_failed( summary="Found deleted Subscriptions", severity=Severity.High, - participants=[dss.participant_id], details=f"Deleted Subscriptions found: {found_deleted_sub}", query_timestamps=[subs.query.request.timestamp], ) @@ -532,7 +526,6 @@ def step9(self): check.record_failed( summary=f"DSS returned expired ISA {isa_1.uuid} when creating Subscription {sub_2.uuid}", severity=Severity.High, - participants=[dss.participant_id], details=f"service_areas IDs: {', '.join(isa_ids)}", query_timestamps=[created_sub.query.request.timestamp], ) @@ -563,7 +556,6 @@ def step10(self): check.record_failed( summary=f"DSS returned too few Subscriptions", severity=Severity.High, - participants=[self._dss_primary.participant_id], details=f"Missing Subscriptions: {', '.join(missing_subs)}", query_timestamps=[mutated_isa.dss_query.query.request.timestamp], ) @@ -591,7 +583,6 @@ def step11(self): check.record_failed( summary=f"DSS returned too few Subscriptions", severity=Severity.High, - participants=[self._dss_primary.participant_id], details=f"Missing Subscriptions: {', '.join(missing_subs)}", query_timestamps=[del_isa.dss_query.query.request.timestamp], ) @@ -628,7 +619,6 @@ def step12(self): check.record_failed( summary="Found expired Subscriptions", severity=Severity.High, - participants=[self._dss_primary.participant_id], details=f"Expired Subscriptions found: {', '.join(found_expired_sub)}", query_timestamps=[mutated_isa.dss_query.query.request.timestamp], ) @@ -658,7 +648,6 @@ def step13(self): check.record_failed( summary="Found expired Subscriptions", severity=Severity.High, - participants=[dss.participant_id], details=f"Expired Subscriptions found: {', '.join(found_expired_sub)}", query_timestamps=[subs.query.request.timestamp], ) @@ -702,7 +691,6 @@ def step15(self): check.record_failed( summary="Found expired Subscriptions", severity=Severity.High, - participants=[self._dss_primary.participant_id], details=f"Expired Subscriptions found: {', '.join(found_expired_sub)}", query_timestamps=[del_isa.dss_query.query.request.timestamp], ) @@ -734,7 +722,6 @@ def step16(self): check.record_failed( summary=f"DSS returned expired ISA {isa_3.uuid} when creating Subscription {sub_3.uuid}", severity=Severity.High, - participants=[dss.participant_id], details=f"service_areas IDs: {', '.join(isa_ids)}", query_timestamps=[created_sub.query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index 76411d768d..6474070e69 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -184,14 +184,12 @@ def _evaluate_and_test_authentication( if uss_flights_query.success: check.record_failed( "Unauthenticated request for flights to USS was fulfilled", - participants=[participant_id], severity=Severity.Medium, details=f"Queried flights on {flights_url} for USS {participant_id} with no credentials, expected a failure but got a success reply.", ) elif uss_flights_query.status_code != 401: check.record_failed( "Unauthenticated request for flights failed with wrong HTTP code", - participants=[participant_id], severity=Severity.Medium, details=f"Queried flights on {flights_url} for USS {participant_id} with no credentials, expected an HTTP 401 but got an HTTP {uss_flights_query.status_code}.", ) @@ -211,14 +209,12 @@ def _evaluate_and_test_authentication( if uss_flight_details_query.success: check.record_failed( "Unauthenticated request for flight details to USS was fulfilled", - participants=[participant_id], severity=Severity.Medium, details=f"Queried flight details on {flights_url} for USS {participant_id} for flight {flight.id} with no credentials, expected a failure but got a success reply.", ) elif uss_flight_details_query.status_code != 401: check.record_failed( "Unauthenticated request for flight details failed with wrong HTTP code", - participants=[participant_id], severity=Severity.Medium, details=f"Queried flight details on {flights_url} for USS {participant_id} for flight {flight.id} with no credentials, expected an HTTP 401 but got an HTTP {uss_flight_details_query.status_code}.", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 33aacbb912..f1795ceea7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -820,7 +820,6 @@ def _evaluate_sp_observation( summary="Could not query ISAs from DSS", severity=Severity.Medium, details=f"Query to {self._dss.participant_id}'s DSS at {sp_observation.dss_isa_query.query.request.url} failed {sp_observation.dss_isa_query.query.status_code}", - participants=[self._dss.participant_id], query_timestamps=[ sp_observation.dss_isa_query.query.request.initiated_at.datetime ], @@ -936,7 +935,6 @@ def _evaluate_normal_sp_observation( summary="Flight details query not successful", severity=Severity.Medium, details=f"Flight details query to {details_query.query.request.url} failed {details_query.status_code}", - participants=[mapping.injected_flight.uss_participant_id], query_timestamps=[details_query.query.request.timestamp], ) errors = schema_validation.validate( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py index df647c714b..52796fada7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py @@ -114,7 +114,6 @@ def _op_intent_details_step(self): if p95 > constants.MaxRespondToOIDetailsRequestSeconds: check.record_failed( summary=f"95th percentile of durations for operational intent details requests to USS is higher than threshold", - participants=[participant], details=f"threshold: {constants.MaxRespondToOIDetailsRequestSeconds}s, 95th percentile: {p95}s", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py index be9b622562..72cd0bc586 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py @@ -69,7 +69,6 @@ def _test_env_reqs(self): elif ipaddress.ip_address(ip_addr).is_private: check.record_failed( summary=f"DSS host {parsed_url.netloc} is not publicly addressable", - participants=[dss.participant_id], details=f"DSS (URL: {dss.base_url}, netloc: {parsed_url.netloc}, resolved IP: {ip_addr}) is not publicly addressable", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index aaa242d3c9..62226e5802 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -196,12 +196,13 @@ def _validate_ended_cancellation(self): "Validate flight intent shared correctly", self._intents_extent, ) as planned_validator: + self.begin_test_step("Plan flight intent") _, flight_id = plan_flight_intent( self, - "Plan flight intent", self.tested_uss, self.valid_flight.request, ) + self.end_test_step() planned_validator.expect_shared(self.valid_flight.request) _ = delete_flight_intent( @@ -210,12 +211,13 @@ def _validate_ended_cancellation(self): cancelled_validator.expect_not_shared() def _validate_precision_intersection(self): + self.begin_test_step("Plan control flight intent") _, _ = plan_flight_intent( self, - "Plan control flight intent", self.tested_uss, self.valid_flight.request, ) + self.end_test_step() with OpIntentValidator( self, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index 4e24bbabbb..d7b880f4aa 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -214,12 +214,13 @@ def _attempt_plan_flight_conflict(self) -> OperationalIntentReference: "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") _, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -246,12 +247,13 @@ def _attempt_plan_flight_conflict(self) -> OperationalIntentReference: "Validate Flight 1 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to plan Flight 1") plan_conflict_flight_intent( self, - "Attempt to plan Flight 1", self.tested_uss, self.flight1_planned.request, ) + self.end_test_step() validator.expect_not_shared() return flight_2_oi_ref @@ -264,13 +266,14 @@ def _attempt_activate_flight_conflict(self): "Validate Flight 1 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to directly activate conflicting Flight 1") activate_conflict_flight_intent( self, - "Attempt to directly activate conflicting Flight 1", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) + self.end_test_step() validator.expect_not_shared() def _attempt_modify_planned_flight_conflict( @@ -283,12 +286,13 @@ def _attempt_modify_planned_flight_conflict( "Validate Flight 1c sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 1c") _, self.flight1_id = plan_flight_intent( self, - "Plan Flight 1c", self.tested_uss, self.flight1c_planned.request, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared(self.flight1c_planned.request) with OpIntentValidator( @@ -299,13 +303,14 @@ def _attempt_modify_planned_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify planned Flight 1c into conflict") modify_planned_conflict_flight_intent( self, - "Attempt to modify planned Flight 1c into conflict", self.tested_uss, self.flight1_planned.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1c_planned.request, skip_if_not_found=True ) @@ -340,13 +345,14 @@ def _attempt_modify_activated_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify activated Flight 1c into conflict") modify_activated_conflict_flight_intent( self, - "Attempt to modify activated Flight 1c into conflict", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1c_activated.request, skip_if_not_found=True ) @@ -386,12 +392,13 @@ def _modify_activated_flight_preexisting_conflict( "Validate Flight 2m sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2m") _, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2m", self.control_uss, self.flight2m_planned.request, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2m_planned.request) with OpIntentValidator( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py index 32fd8e6ad6..3628c0988b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py @@ -214,12 +214,13 @@ def _attempt_plan_flight_conflict(self): "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") resp_flight_2, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -229,12 +230,13 @@ def _attempt_plan_flight_conflict(self): "Validate Flight 1 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to plan Flight 1") _ = plan_priority_conflict_flight_intent( self, - "Attempt to plan Flight 1", self.tested_uss, self.flight1_planned.request, ) + self.end_test_step() validator.expect_not_shared() _ = delete_flight_intent( @@ -252,12 +254,13 @@ def _attempt_modify_planned_flight_conflict( "Validate Flight 1 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 1") resp_flight_1, self.flight1_id = plan_flight_intent( self, - "Plan Flight 1", self.tested_uss, self.flight1_planned.request, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared(self.flight1_planned.request) with OpIntentValidator( @@ -267,12 +270,13 @@ def _attempt_modify_planned_flight_conflict( "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") resp_flight_2, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -283,13 +287,14 @@ def _attempt_modify_planned_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify planned Flight 1 in conflict") _ = modify_planned_priority_conflict_flight_intent( self, - "Attempt to modify planned Flight 1 in conflict", self.tested_uss, self.flight1m_planned.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1_planned.request, skip_if_not_found=True ) @@ -307,13 +312,14 @@ def _attempt_activate_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to activate conflicting Flight 1") _ = activate_priority_conflict_flight_intent( self, - "Attempt to activate conflicting Flight 1", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1_planned.request, skip_if_not_found=True ) @@ -352,12 +358,13 @@ def _modify_activated_flight_conflict_preexisting( "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") _, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -436,13 +443,14 @@ def _attempt_modify_activated_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify activated Flight 1 in conflict") modify_activated_priority_conflict_flight_intent( self, - "Attempt to modify activated Flight 1 in conflict", self.tested_uss, self.flight1c_activated.request, self.flight1_id, ) + self.end_test_step() validator.expect_shared( flight_1_intent.request, skip_if_not_found=True, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py index 647f4f4be4..ca704c6927 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py @@ -144,9 +144,9 @@ def _setup(self): ) self.end_test_step() - set_uss_available( - self, "Restore virtual USS availability", self.dss, self.uss_qualifier_sub - ) + self.begin_test_step("Restore virtual USS availability") + set_uss_available(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() self.begin_test_step("Clear operational intents created by virtual USS") self._clear_op_intents() @@ -216,9 +216,9 @@ def _plan_flight_conflict_planned(self): ) # Declare virtual USS as down at DSS test step - set_uss_down( - self, "Declare virtual USS as down at DSS", self.dss, self.uss_qualifier_sub - ) + self.begin_test_step("Declare virtual USS as down at DSS") + set_uss_down(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() # Tested USS attempts to plan Flight 1 test step with OpIntentValidator( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py index 6cb4095653..315843982d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py @@ -90,9 +90,9 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference: ) # Declare virtual USS as down at DSS test step - set_uss_down( - self, "Declare virtual USS as down at DSS", self.dss, self.uss_qualifier_sub - ) + self.begin_test_step("Declare virtual USS as down at DSS") + set_uss_down(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() # Tested USS attempts to plan high-priority flight 2 test step with OpIntentValidator( @@ -121,12 +121,9 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference: validator.expect_not_shared() # Restore virtual USS availability at DSS test step - set_uss_available( - self, - "Restore virtual USS availability at DSS", - self.dss, - self.uss_qualifier_sub, - ) + self.begin_test_step("Restore virtual USS availability at DSS") + set_uss_available(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() return oi_ref @@ -140,9 +137,9 @@ def _plan_flight_conflict_nonconforming( ) # Declare virtual USS as down at DSS test step - set_uss_down( - self, "Declare virtual USS as down at DSS", self.dss, self.uss_qualifier_sub - ) + self.begin_test_step("Declare virtual USS as down at DSS") + set_uss_down(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() # Tested USS attempts to plan high-priority flight 2 test step with OpIntentValidator( @@ -171,12 +168,9 @@ def _plan_flight_conflict_nonconforming( validator.expect_not_shared() # Restore virtual USS availability at DSS test step - set_uss_available( - self, - "Restore virtual USS availability at DSS", - self.dss, - self.uss_qualifier_sub, - ) + self.begin_test_step("Restore virtual USS availability at DSS") + set_uss_available(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() return oi_ref @@ -188,9 +182,9 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference): ) # Declare virtual USS as down at DSS test step - set_uss_down( - self, "Declare virtual USS as down at DSS", self.dss, self.uss_qualifier_sub - ) + self.begin_test_step("Declare virtual USS as down at DSS") + set_uss_down(self, self.dss, self.uss_qualifier_sub) + self.end_test_step() # Tested USS attempts to plan high-priority flight 2 test step with OpIntentValidator( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index 532dcb632e..3a3a7fd8f2 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -580,18 +580,16 @@ def __eq__(self, other): def set_uss_available( scenario: TestScenarioType, - test_step: str, dss: DSSInstance, uss_sub: str, ) -> str: """Set the USS availability to 'Available'. - This function implements the test step described in set_uss_available.md. + This function implements the test step fragment described in set_uss_available.md. Returns: The new version of the USS availability. """ - scenario.begin_test_step(test_step) availability_version, avail_query = dss.set_uss_availability( uss_sub, True, @@ -606,24 +604,21 @@ def set_uss_available( details=f"DSS responded code {avail_query.status_code}; error message: {avail_query.error_message}", query_timestamps=[avail_query.request.timestamp], ) - scenario.end_test_step() return availability_version def set_uss_down( scenario: TestScenarioType, - test_step: str, dss: DSSInstance, uss_sub: str, ) -> str: """Set the USS availability to 'Down'. - This function implements the test step described in set_uss_down.md. + This function implements the test step fragment described in set_uss_down.md. Returns: The new version of the USS availability. """ - scenario.begin_test_step(test_step) availability_version, avail_query = dss.set_uss_availability( uss_sub, False, @@ -638,5 +633,4 @@ def set_uss_down( details=f"DSS responded code {avail_query.status_code}; error message: {avail_query.error_message}", query_timestamps=[avail_query.request.timestamp], ) - scenario.end_test_step() return availability_version diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py index 52f2357930..253e2dc117 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py @@ -19,7 +19,6 @@ def plan_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> InjectFlightResponse: @@ -30,11 +29,8 @@ def plan_priority_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly planned", @@ -47,13 +43,11 @@ def plan_priority_conflict_flight_intent( flight_intent, ) - scenario.end_test_step() return resp def modify_planned_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -65,11 +59,8 @@ def modify_planned_priority_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -83,13 +74,11 @@ def modify_planned_priority_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def activate_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -102,10 +91,9 @@ def activate_priority_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly activated", @@ -119,13 +107,11 @@ def activate_priority_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -138,10 +124,9 @@ def modify_activated_priority_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -155,13 +140,11 @@ def modify_activated_priority_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def plan_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> InjectFlightResponse: @@ -172,11 +155,8 @@ def plan_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly planned", @@ -189,13 +169,11 @@ def plan_conflict_flight_intent( flight_intent, ) - scenario.end_test_step() return resp def modify_planned_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -207,11 +185,8 @@ def modify_planned_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -225,13 +200,11 @@ def modify_planned_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def activate_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -244,10 +217,9 @@ def activate_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly activated", @@ -261,13 +233,11 @@ def activate_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -280,10 +250,9 @@ def modify_activated_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -297,13 +266,11 @@ def modify_activated_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def plan_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> Tuple[InjectFlightResponse, Optional[str]]: @@ -312,15 +279,14 @@ def plan_permitted_conflict_flight_intent( This function implements the test step described in plan_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0055. + TODO: Remove this function if it is not used in the future + Returns: * The injection response. * The ID of the injected flight if it is returned, None otherwise. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, flight_id = submit_flight_intent( scenario, "Successful planning", @@ -330,13 +296,11 @@ def plan_permitted_conflict_flight_intent( flight_intent, ) - scenario.end_test_step() return resp, flight_id def modify_planned_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -346,13 +310,12 @@ def modify_planned_permitted_conflict_flight_intent( This function implements the test step described in modify_planned_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0060. + TODO: Remove this function if it is not used in the future + Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Successful modification", @@ -363,13 +326,11 @@ def modify_planned_permitted_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def activate_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -379,13 +340,14 @@ def activate_permitted_conflict_flight_intent( This function implements the test step described in activate_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0065. + TODO: Remove this function if it is not used in the future + Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Successful activation", @@ -396,13 +358,11 @@ def activate_permitted_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -412,13 +372,14 @@ def modify_activated_permitted_conflict_flight_intent( This function implements the test step described in modify_activated_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0070. + TODO: Remove this function if it is not used in the future + Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Successful modification", @@ -429,5 +390,4 @@ def modify_activated_permitted_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 5324b63f2a..c7f7317a7e 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -36,11 +36,11 @@ def expect_flight_intent_state( flight_intent: InjectFlightRequest, expected_state: OperationalIntentState, scenario: TestScenarioType, - test_step: str, ) -> None: """Confirm that provided flight intent test data has the expected state or raise a ValueError.""" if flight_intent.operational_intent.state != expected_state: function_name = str(inspect.stack()[1][3]) + test_step = scenario.current_step_name() raise ValueError( f"Error in test data: operational intent state for {function_name} during test step '{test_step}' in scenario '{scenario.documentation.name}' is expected to be `{expected_state}`, but got `{flight_intent.operational_intent.state}` instead" ) @@ -48,7 +48,6 @@ def expect_flight_intent_state( def plan_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> Tuple[InjectFlightResponse, Optional[str]]: @@ -62,11 +61,8 @@ def plan_flight_intent( * The injection response. * The ID of the injected flight if it is returned, None otherwise. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, flight_id = submit_flight_intent( scenario, "Successful planning", @@ -76,7 +72,6 @@ def plan_flight_intent( flight_intent, ) - scenario.end_test_step() return resp, flight_id @@ -95,7 +90,7 @@ def activate_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) scenario.begin_test_step(test_step) @@ -127,9 +122,7 @@ def modify_planned_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( @@ -163,7 +156,7 @@ def modify_activated_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) scenario.begin_test_step(test_step) diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index f95fca11ca..ccdec94961 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -242,6 +242,12 @@ def cleanup(self): def me(self) -> str: return inspection.fullname(self.__class__) + def current_step_name(self) -> Optional[str]: + if self._current_step: + return self._current_step.name + else: + return None + def _make_scenario_report(self) -> None: self._scenario_report = TestScenarioReport( name=self.documentation.name, diff --git a/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py b/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py index 4a20805057..2c1e1182cb 100644 --- a/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py +++ b/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py @@ -78,8 +78,10 @@ def run(self, context: ExecutionContext): self.end_test_case() self.begin_test_case("Plan valid flight") + self.begin_test_step("Plan valid flight intent") if not self._plan_valid_flight(): return + self.end_test_step() self.end_test_case() self.end_test_scenario() @@ -135,7 +137,6 @@ def _attempt_invalid_flights(self) -> bool: def _plan_valid_flight(self) -> bool: resp, _ = plan_flight_intent( self, - "Plan valid flight intent", self.ussp, self.valid_flight_intent.request, ) diff --git a/monitoring/uss_qualifier/scenarios/uspace/netrid/__init__.py b/monitoring/uss_qualifier/scenarios/uspace/netrid/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md new file mode 100644 index 0000000000..1307c69742 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md @@ -0,0 +1,48 @@ +# U-space MSL altitude test scenario + +## Description + +[Article 8](https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:32021R0664&qid=1702917443967#d1e905-161-1)(2)(c) specifies (emphasis added): + +> The network identification service shall allow for the authorised users to receive messages with the geographical position of the UAS, its *altitude above mean sea level* and its height above the surface or take-off point. + + +ASTM F3411 provides geodetic altitude above the WGS84 ellipsoidal estimate of sea level, however [AMC1](https://www.easa.europa.eu/en/document-library/acceptable-means-of-compliance-and-guidance-materials/amc-and-gm-implementing) specifies: + +> USSPs should convert the heights above the WGS 84 ellipsoid exchanged with the ASTM F-3411-22A standard to height above mean sea level (MSL) before providing it to the UAS operators. + + +[GM1](https://www.easa.europa.eu/en/document-library/acceptable-means-of-compliance-and-guidance-materials/amc-and-gm-implementing) further clarifies the desired definition of mean sea level: + +> Wherever the flight altitude above sea level is required to be determined with the use of GNSS systems, it is recommended to use the EGM2008 or at least the EGM96 geoid models as the definition of mean sea level, as agreed with the competent authority. + + +Therefore, to comply with Article 8(2)(c), a USSP must allow for the authorised users to receive messages with the UAS's altitude above the EGM96 geoid. + +### Assumptions + +This scenario assumes that [the ASTM F3411-22a nominal behavior NetRID test scenario](../../astm/netrid/v22a/nominal_behavior.md) has already been completed and determines compliance with the requirement above by examining the observations made by uss_qualifier as an automated "authorised user". + +## Resources + +### observers + +The set of USSPs providing messages to authorised users to be evaluated for U-space MSL compliance. + +## UAS observations evaluation test case + +### Find nominal behavior report test step + +To avoid re-running a nearly-identical test scenario, this test scenario merely examines data collected in a separate test scenario (see [Assumptions](#assumptions)). Therefore, the first step in this scenario is to find the test report for that other scenario. + +If an appropriate test report cannot be found, this scenario will be discontinued. + +### Evaluate UAS observations test step + +#### ⚠️ Message contains MSL altitude check + +If the response message for the remote identification observation made by the virtual/automated authorised user does not contain the UAS's MSL altitude, the USSP will have failed to comply with **[uspace.article8.MSLAltitude](../../../requirements/uspace/article8.md)**. + +#### ⚠️ MSL altitude is correct check + +In the previously-conducted test scenario, UAS altitudes were injected relative to the WGS84 ellipsoid. Since the EGM96 geoid is a standard shape that is well-defined relative to the WGS84 ellipsoid, this means the altitude relative to the EGM96 is defined by the injection. If the observed MSL altitude differs from the injected MSL altitude, then the USSP has failed to allow the automated authorised user to receive messages with the UAS's altitude above mean sea level per **[uspace.article8.MSLAltitude](../../../requirements/uspace/article8.md)** because the altitude reported was not the altitude of the UAS. diff --git a/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py new file mode 100644 index 0000000000..c90277688c --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py @@ -0,0 +1,31 @@ +from typing import List + +from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.resources.netrid import NetRIDObserversResource +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class MSLAltitude(TestScenario): + _ussps: List[ParticipantID] + + def __init__(self, observers: NetRIDObserversResource): + super().__init__() + self._ussps = [obs.participant_id for obs in observers.observers] + + def run(self, context): + self.begin_test_scenario(context) + + self.begin_test_case("UAS observations evaluation") + + self.begin_test_step("Find nominal behavior report") + # TODO: Find test report for NetRID nominal behavior scenario + self.end_test_step() + + self.begin_test_step("Evaluate UAS observations") + # TODO: Examine observation queries in test report to see if MSL was present + # TODO: When MSL is present, verify that its value matches injected altitude above ellipsoid + self.end_test_step() + + self.end_test_case() + + self.end_test_scenario() diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index e3a48c7084..3e3f7efd74 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -5,6 +5,7 @@ ## [Actions](../README.md#actions) 1. Suite: [ASTM F3411-22a](../astm/netrid/f3411_22a.md) ([`suites.astm.netrid.f3411_22a`](../astm/netrid/f3411_22a.yaml)) +2. Scenario: [U-space MSL altitude](../../scenarios/uspace/netrid/msl.md) ([`scenarios.uspace.netrid.msl.MSLAltitude`](../../scenarios/uspace/netrid/msl.py)) ## [Checked requirements](../README.md#checked-requirements) @@ -489,4 +490,10 @@ Implemented ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Token Validation
ASTM NetRID nominal behavior + + uspace
.article8 + MSLAltitude + Implemented + U-space MSL altitude + diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.yaml b/monitoring/uss_qualifier/suites/uspace/network_identification.yaml index c786d1edd0..d07ce5578e 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.yaml +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.yaml @@ -22,6 +22,11 @@ actions: id_generator: id_generator service_area: service_area problematically_big_area: problematically_big_area + on_failure: Abort +- test_scenario: + scenario_type: scenarios.uspace.netrid.msl.MSLAltitude + resources: + observers: observers on_failure: Continue participant_verifiable_capabilities: - id: uspace_netrid_service_provider diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index debcace35a..242091f261 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -675,6 +675,12 @@ Implemented Data Validation of GET operational intents by USS + + uspace
.article8 + MSLAltitude + Implemented + U-space MSL altitude + versioning ReportSystemVersion