diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 973a7dd0a3..93f9bdb0c1 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -15,7 +15,7 @@ from implicitdict import ImplicitDict, StringBasedDateTime from monitoring.monitorlib import infrastructure - +from monitoring.monitorlib.rid import RIDVersion TIMEOUTS = (5, 5) # Timeouts of `connect` and `read` in seconds ATTEMPTS = ( @@ -212,6 +212,15 @@ class QueryType(str, Enum): ) F3548v21USSMakeUssReport = "astm.f3548.v21.uss.makeUssReport" + @staticmethod + def flight_details(rid_version: RIDVersion): + if rid_version == RIDVersion.f3411_19: + return QueryType.F3411v19aFlightDetails + elif rid_version == RIDVersion.f3411_22a: + return QueryType.F3411v22aFlightDetails + else: + raise ValueError(f"Unsupported RID version: {rid_version}") + class Query(ImplicitDict): request: RequestDescription diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index c24023f49b..196409a752 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -56,15 +56,6 @@ def openapi_put_isa_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") - @property - def read_scope(self) -> str: - if self == RIDVersion.f3411_19: - return v19.constants.Scope.Read - elif self == RIDVersion.f3411_22a: - return v22a.constants.Scope.DisplayProvider - else: - raise ValueError("Unsupported RID version '{}'".format(self)) - @property def realtime_period(self) -> timedelta: if self == RIDVersion.f3411_19: @@ -155,6 +146,24 @@ def dp_data_resp_percentile95_s(self) -> float: else: raise ValueError("Unsupported RID version '{}'".format(self)) + @property + def dp_details_resp_percentile95_s(self) -> float: + if self == RIDVersion.f3411_19: + return v19.constants.NetDpDetailsResponse95thPercentileSeconds + elif self == RIDVersion.f3411_22a: + return v22a.constants.NetDpDetailsResponse95thPercentileSeconds + else: + raise ValueError("Unsupported RID version '{}'".format(self)) + + @property + def dp_details_resp_percentile99_s(self) -> float: + if self == RIDVersion.f3411_19: + return v19.constants.NetDpDetailsResponse99thPercentileSeconds + elif self == RIDVersion.f3411_22a: + return v22a.constants.NetDpDetailsResponse99thPercentileSeconds + else: + raise ValueError("Unsupported RID version '{}'".format(self)) + @property def dp_data_resp_percentile99_s(self) -> float: if self == RIDVersion.f3411_19: diff --git a/monitoring/uss_qualifier/resources/netrid/observers.py b/monitoring/uss_qualifier/resources/netrid/observers.py index da330d3651..f335996e06 100644 --- a/monitoring/uss_qualifier/resources/netrid/observers.py +++ b/monitoring/uss_qualifier/resources/netrid/observers.py @@ -5,6 +5,7 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import fetch, infrastructure +from monitoring.monitorlib.fetch import QueryType from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.rid import RIDVersion from monitoring.monitorlib.rid_automated_testing import observation_api @@ -27,9 +28,6 @@ def __init__( self.participant_id = participant_id self.base_url = base_url - # TODO: Change observation API to use an InterUSS scope rather than re-using an ASTM scope - self.rid_version = RIDVersion.f3411_19 - def observe_system( self, rect: s2sphere.LatLngRect ) -> Tuple[Optional[observation_api.GetDisplayDataResponse], fetch.Query]: @@ -40,7 +38,12 @@ def observe_system( rect.hi().lng().degrees, ) query = fetch.query_and_describe( - self.session, "GET", url, scope=self.rid_version.read_scope + self.session, + "GET", + url, + # TODO replace with 'uas_standards.interuss.automated_testing.rid.v1.constants.Scope.Observe' once + # the standard is updated with https://github.com/interuss/uas_standards/pull/11/files + scope="dss.read.identification_service_areas", ) try: result = ( @@ -56,11 +59,19 @@ def observe_system( return result, query def observe_flight_details( - self, flight_id: str + self, flight_id: str, rid_version: RIDVersion ) -> Tuple[Optional[observation_api.GetDetailsResponse], fetch.Query]: query = fetch.query_and_describe( - self.session, "GET", f"/display_data/{flight_id}" + self.session, + "GET", + f"/display_data/{flight_id}", + # TODO replace with 'uas_standards.interuss.automated_testing.rid.v1.constants.Scope.Observe' once + # the standard is updated with https://github.com/interuss/uas_standards/pull/11/files + scope="dss.read.identification_service_areas", ) + # Record query metadata for later use in the aggregate checks + query.server_id = self.participant_id + query.query_type = QueryType.flight_details(rid_version) try: result = ( ImplicitDict.parse( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index 909227ffff..f157b8bbc2 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -92,6 +92,10 @@ def run(self): self._dp_display_data_times_step() self.end_test_step() + self.begin_test_step("Performance of /display_data/ requests") + self._dp_display_data_details_times_step() + self.end_test_step() + self.end_test_case() # SP performance @@ -105,6 +109,60 @@ def run(self): self.end_test_scenario() + def _dp_display_data_details_times_step(self): + """ + Check performance of /display_data/ requests and confirm they conform to + NetDpDetailsResponse95thPercentile (2s) and NetDpDetailsResponse99thPercentile (6s) + """ + for participant, all_queries in self._queries_by_participant.items(): + relevant_queries: List[fetch.Query] = list() + for query in all_queries: + + if ( + query.status_code == 200 + and query.has_field_with_value("query_type") + and ( + query.query_type == QueryType.F3411v19aFlightDetails + or query.query_type == QueryType.F3411v22aFlightDetails + ) + ): + relevant_queries.append(query) + + if len(relevant_queries) == 0: + # this may be a service provider + self.record_note( + f"{participant}/display_data/", + "skipped check: no relevant queries", + ) + continue + + # compute percentiles + durations = [query.response.elapsed_s for query in relevant_queries] + [p95, p99] = evaluation.compute_percentiles(durations, [95, 99]) + with self.check( + "Performance of /display_data/ requests", [participant] + ) as check: + if p95 > self._rid_version.dp_details_resp_percentile95_s: + check.record_failed( + summary=f"95th percentile of durations for DP display_data details queries is higher than threshold", + severity=Severity.Medium, + participants=[participant], + details=f"threshold: {self._rid_version.dp_details_resp_percentile95_s}s, 95th percentile: {p95}s", + ) + if p99 > self._rid_version.dp_details_resp_percentile99_s: + check.record_failed( + summary=f"99th percentile of durations for DP display_data details queries is higher than threshold", + severity=Severity.Medium, + participants=[participant], + details=f"threshold: {self._rid_version.dp_details_resp_percentile99_s}s, 99th percentile: {p99}s", + ) + + self.record_note( + f"{participant}/display_data/", + f"{participant}/display_data/ stats computed on {len(durations)} queries " + f"95th percentile: {p95}s, 99th percentile: {p99}s", + ) + def _sp_flights_area_times_step(self): for participant, all_queries in self._queries_by_participant.items(): # identify successful flights queries diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index c069a0beaa..67765986bb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -10,7 +10,7 @@ RIDCommonDictionaryEvaluator, ) -from monitoring.monitorlib.fetch import Query +from monitoring.monitorlib.fetch import Query, QueryType from monitoring.monitorlib.fetch.rid import ( all_flights, FetchedFlights, @@ -261,6 +261,16 @@ def evaluate_system_instantaneously( query, verified_sps, ) + # We also issue queries to the flight details endpoint in order to collect + # performance statistics, which are computed and checked at a later stage. + if query.status_code == 200: + # If there are multiple flights, we only issue a single details query for the first returned one, + # as we don't want to slow down the test we are piggy-backing on. + if len(observation.flights) > 0: + (_, detailQuery) = observer.observe_flight_details( + observation.flights[0].id, self._rid_version + ) + self._test_scenario.record_query(detailQuery) # TODO: If bounding rect is smaller than cluster threshold, expand slightly above cluster threshold and re-observe # TODO: If bounding rect is smaller than area-too-large threshold, expand slightly above area-too-large threshold and re-observe @@ -617,14 +627,14 @@ def _evaluate_clusters_observation( check.record_failed( summary="Error while evaluating clustered area view. Missing flight", severity=Severity.Medium, - details=f"{expected_count-clustered_flight_count} (~{uncertain_count}) missing flight(s)", + details=f"{expected_count - clustered_flight_count} (~{uncertain_count}) missing flight(s)", ) elif clustered_flight_count > expected_count + uncertain_count: # Unexpected flight check.record_failed( summary="Error while evaluating clustered area view. Unexpected flight", severity=Severity.Medium, - details=f"{clustered_flight_count-expected_count} (~{uncertain_count}) unexpected flight(s)", + details=f"{clustered_flight_count - expected_count} (~{uncertain_count}) unexpected flight(s)", ) elif clustered_flight_count == expected_count: # evaluate cluster obfuscation distance @@ -674,7 +684,7 @@ def _evaluate_clusters_obfuscation_distance( cluster_width, cluster_height = geo.flatten( cluster_rect.lo(), cluster_rect.hi() ) - min_dim = 2 * observer.rid_version.min_obfuscation_distance_m + min_dim = 2 * self._rid_version.min_obfuscation_distance_m if cluster_height < min_dim or cluster_width < min_dim: # Cluster has a too small distance to the edge check.record_failed( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 8dcbdfead6..598b3b52c0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -17,6 +17,16 @@ The observers to evaluate in the report. ## Performance of Display Providers requests test case +### Performance of /display_data/ requests test step + +For this step, all successful display data queries made during the execution of the previous scenarios are used to compute an aggregate statistic. + +#### Performance of /display_data/ requests check + +**[astm.f3411.v19.NET0460](../../../../requirements/astm/f3411/v19.md) Checks that the DP response times for the +Display Application's flight details requests have a p95 and p99 that are respectively below +`NetDpDetailsResponse95thPercentileSeconds` (2 seconds) and `NetDpDetailsResponse99thPercentileSeconds` (6 seconds). + ### Performance of /display_data requests test step In this step, all successful display data queries made during the execution of the previous scenarios are aggregated per observer and per request (identified by their URLs). For each of those, and using the session length diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 17760ea7eb..94e1b5482c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -4,7 +4,6 @@ In this special scenario, the report of previously executed ASTM F3411-22a NetRID scenario(s) are evaluated for the performances of the queries made during their execution. - ## Resources ### report_resource @@ -18,6 +17,16 @@ The observers to evaluate in the report. ## Performance of Display Providers requests test case +### Performance of /display_data/ requests test step + +For this step, all successful display data queries made during the execution of the previous scenarios are used to compute an aggregate statistic. + +#### Performance of /display_data/ requests check + +**[astm.f3411.v22a.NET0460](../../../../requirements/astm/f3411/v22a.md) Checks that the DP response times for the +Display Application's flight details requests have a p95 and p99 that are respectively below +`NetDpDetailsResponse95thPercentileSeconds` (2 seconds) and `NetDpDetailsResponse99thPercentileSeconds` (6 seconds). + ### Performance of /display_data requests test step In this step, all successful display data queries made during the execution of the previous scenarios are aggregated per observer and per request (identified by their URLs). For each of those, and using the session length