[uss_qualifier] NET0260-a performance check for querying flights in an area from a SP

monitoring/monitorlib/fetch/__init__.py

@@ -4,6 +4,8 @@
 import traceback
 from typing import Dict, Optional, List
 
+from enum import Enum
+
 import flask
 from loguru import logger
 import requests
@@ -137,10 +139,23 @@ def describe_response(resp: requests.Response) -> ResponseDescription:
     return ResponseDescription(**kwargs)
 
 
+class QueryType(str, Enum):
+    F3411v22aFlights = "astm.f3411.v22a.sp.flights"
+    F3411v19Flights = "astm.f3411.v19.sp.flights"
+    F3411v22aFlightDetails = "astm.f3411.v22a.sp.flight_details"
+    F3411v19aFlightDetails = "astm.f3411.v19.sp.flight_details"
+
+
 class Query(ImplicitDict):
     request: RequestDescription
     response: ResponseDescription
 
+    server_id: Optional[str]
+    """If specified, identifier of the USS/participant hosting the server involved in this query."""
+
+    query_type: Optional[QueryType]
+    """If specified, the recognized type of this query."""
+
     @property
     def status_code(self) -> int:
         return self.response.status_code

monitoring/monitorlib/fetch/rid.py

@@ -13,7 +13,7 @@
 from yaml.representer import Representer
 
 from monitoring.monitorlib import fetch, rid_v1, rid_v2, geo
-from monitoring.monitorlib.fetch import Query
+from monitoring.monitorlib.fetch import Query, QueryType
 from monitoring.monitorlib.infrastructure import UTMClientSession
 from monitoring.monitorlib.rid import RIDVersion
 
@@ -452,6 +452,14 @@ def success(self) -> bool:
     def errors(self) -> List[str]:
         raise NotImplementedError("RIDQuery.errors must be overriden")
 
+    def set_server_id(self, server_id: str):
+        if self.v19_query is not None:
+            self.v19_query.server_id = server_id
+        elif self.v22a_query is not None:
+            self.v22a_query.server_id = server_id
+        else:
+            raise NotImplementedError(f"Cannot set server_id")
+
 
 class FetchedISA(RIDQuery):
     """Version-independent representation of an ISA read from the DSS."""
@@ -766,6 +774,7 @@ def uss_flights(
             },
             scope=v19.constants.Scope.Read,
         )
+        query.query_type = QueryType.F3411v19Flights
         return FetchedUSSFlights(v19_query=query)
     elif rid_version == RIDVersion.f3411_22a:
         params = {
@@ -785,6 +794,7 @@ def uss_flights(
             params=params,
             scope=v22a.constants.Scope.DisplayProvider,
         )
+        query.query_type = QueryType.F3411v22aFlights
         return FetchedUSSFlights(v22a_query=query)
     else:
         raise NotImplementedError(

monitoring/monitorlib/rid.py

@@ -164,6 +164,24 @@ def dp_data_resp_percentile99_s(self) -> float:
         else:
             raise ValueError("Unsupported RID version '{}'".format(self))
 
+    @property
+    def sp_data_resp_percentile95_s(self) -> float:
+        if self == RIDVersion.f3411_19:
+            return v19.constants.NetSpDataResponseTime95thPercentileSeconds
+        elif self == RIDVersion.f3411_22a:
+            return v22a.constants.NetSpDataResponseTime95thPercentileSeconds
+        else:
+            raise ValueError("Unsupported RID version '{}'".format(self))
+
+    @property
+    def sp_data_resp_percentile99_s(self) -> float:
+        if self == RIDVersion.f3411_19:
+            return v19.constants.NetSpDataResponseTime99thPercentileSeconds
+        elif self == RIDVersion.f3411_22a:
+            return v22a.constants.NetSpDataResponseTime99thPercentileSeconds
+        else:
+            raise ValueError("Unsupported RID version '{}'".format(self))
+
     def flights_url_of(self, base_url: str) -> str:
         if self == RIDVersion.f3411_19:
             return base_url

monitoring/uss_qualifier/resources/netrid/observers.py

@@ -96,7 +96,9 @@ def __init__(
     ):
         self.observers = [
             RIDSystemObserver(
-                o.participant_id, o.observation_base_url, auth_adapter.adapter
+                o.participant_id,
+                o.observation_base_url,
+                auth_adapter.adapter,
             )
             for o in specification.observers
         ]

monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py

@@ -2,7 +2,7 @@
 from typing import List, Dict
 
 from monitoring.monitorlib import fetch
-from monitoring.monitorlib.fetch import evaluation
+from monitoring.monitorlib.fetch import evaluation, QueryType
 from monitoring.monitorlib.rid import RIDVersion
 from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.configurations.configuration import ParticipantID
@@ -60,20 +60,94 @@ def __init__(
                     self._queries_by_participant[participant].append(query)
                     break
 
+            # Only consider queries with the participant/server explicitly identified
+            if query.has_field_with_value("server_id"):
+                participant_queries = self._queries_by_participant.get(
+                    query.server_id, []
+                )
+                participant_queries.append(query)
+                self._queries_by_participant[query.server_id] = participant_queries
+
     def run(self):
         self.begin_test_scenario()
+
         self.record_note("participants", str(self._participants_by_base_url))
         self.record_note("nb_queries", str(len(self._queries)))
 
-        self.begin_test_case("Performance of Display Providers requests")
+        for sp in self._service_providers:
+            self.record_note(
+                "service_providers",
+                f"configured service providers: {sp.participant_id} - {sp.base_url}",
+            )
 
+        for o in self._observers:
+            self.record_note(
+                "observer", f"configured observer: {o.participant_id} - {o.base_url}"
+            )
+
+        # DP performance
+        self.begin_test_case("Performance of Display Providers requests")
         self.begin_test_step("Performance of /display_data requests")
+
         self._dp_display_data_times_step()
+
         self.end_test_step()
+        self.end_test_case()
+
+        # SP performance
+        self.begin_test_case("Performance of Service Providers requests")
+        self.begin_test_step("Performance of /flights?view requests")
+
+        self._sp_flights_area_times_step()
 
+        self.end_test_step()
         self.end_test_case()
+
         self.end_test_scenario()
 
+    def _sp_flights_area_times_step(self):
+        for participant, all_queries in self._queries_by_participant.items():
+            # identify successful flights queries
+            relevant_queries: List[fetch.Query] = list()
+            for query in all_queries:
+                if query.has_field_with_value("query_type") and (
+                    # TODO find a cleaner way than checking for version here
+                    query.query_type == QueryType.F3411v19Flights
+                    or query.query_type == QueryType.F3411v22aFlights
+                ):
+                    relevant_queries.append(query)
+
+            if len(relevant_queries) == 0:
+                # this may be a display provider
+                self.record_note(
+                    f"{participant}/flights", "skipped check: no relevant queries"
+                )
+
+                continue
+
+            # Collect query durations
+            durations = [query.response.elapsed_s for query in relevant_queries]
+            (p95, p99) = evaluation.compute_percentiles(durations, [95, 99])
+
+            with self.check(
+                "Performance for replies to requested flights in an area", [participant]
+            ) as check:
+                if p95 > self._rid_version.sp_data_resp_percentile95_s:
+                    check.record_failed(
+                        f"95th percentile of /flights?view requests is {p95} s, "
+                        f"expected less than {self._rid_version.sp_data_resp_percentile95_s} s"
+                    )
+                if p99 > self._rid_version.sp_data_resp_percentile99_s:
+                    check.record_failed(
+                        f"99th percentile of /flights?view requests is {p99} s, "
+                        f"expected less than {self._rid_version.sp_data_resp_percentile99_s} s"
+                    )
+
+            self.record_note(
+                f"{participant}/flights",
+                f"percentiles on {len(relevant_queries)} relevant queries: 95th: {p95}; 99th: {p99}",
+            )
+
     def _dp_display_data_times_step(self):
         """
         :return: the query durations of respectively the initial queries and the subsequent ones

monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py

@@ -778,6 +778,13 @@ def _evaluate_sp_observation(
             self._injected_flights, observed_flights
         )
 
+        for telemetry_mapping in mapping_by_injection_id.values():
+            # For flights that were mapped to an injection ID,
+            # update the observation queries with the participant id for future use in the aggregate checks
+            telemetry_mapping.observed_flight.query.set_server_id(
+                telemetry_mapping.injected_flight.uss_participant_id
+            )
+
         diagonal_km = (
             rect.lo().get_distance(rect.hi()).degrees * geo.EARTH_CIRCUMFERENCE_KM / 360
         )

monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md

@@ -4,7 +4,6 @@
 In this special scenario, the report of previously executed ASTM F3411-19 NetRID scenario(s) are evaluated for the
 performances of the queries made during their execution.
 
-
 ## Resources
 
 ### report_resource
@@ -16,7 +15,6 @@ The service providers to evaluate in the report.
 ### observers
 The observers to evaluate in the report.
 
-
 ## Performance of Display Providers requests test case
 
 ### Performance of /display_data requests test step
@@ -34,3 +32,13 @@ of the durations for the initial display data queries do not exceed the respecti
 **[astm.f3411.v19.NET0440](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles
 of the durations for the subsequent display data queries do not exceed the respectives thresholds
 `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`.
+
+## Performance of Service Providers requests test case
+
+### Performance of /flights?view requests test step
+
+#### Performance for replies to requested flights in an area check
+
+**[astm.f3411.v19.NET0260-a](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles
+of the durations for the replies to requested flights in an area do not exceed the respective thresholds
+`NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds).

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md

@@ -16,7 +16,6 @@ The service providers to evaluate in the report.
 ### observers
 The observers to evaluate in the report.
 
-
 ## Performance of Display Providers requests test case
 
 ### Performance of /display_data requests test step
@@ -34,3 +33,13 @@ of the durations for the initial display data queries do not exceed the respecti
 **[astm.f3411.v22a.NET0440](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles
 of the durations for the subsequent display data queries do not exceed the respectives thresholds
 `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`.
+
+## Performance of Service Providers requests test case
+
+### Performance of /flights?view requests test step
+
+#### Performance for replies to requested flights in an area check
+
+**[astm.f3411.v22a.NET0260-a](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles
+of the durations for the replies to requested flights in an area do not exceed the respective thresholds
+`NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds).

monitoring/uss_qualifier/scripts/report_analyzer.py

@@ -0,0 +1,77 @@
+import sys
+import json
+
+from implicitdict import ImplicitDict
+
+from monitoring.uss_qualifier.reports.report import (
+    TestRunReport,
+    TestSuiteReport,
+    TestScenarioReport,
+)
+
+
+def parse_report(path: str) -> TestRunReport:
+    with open(path, "r") as f:
+        report = json.load(f)
+        return ImplicitDict.parse(report, TestRunReport)
+
+
+def look_at_test_suite(ts: TestSuiteReport):
+    print("test-suite: " + ts.name)
+
+
+def look_at_scenario(ts: TestScenarioReport):
+    print("Looking at test scenario: ", ts.name)
+    print("Has #cases: ", len(ts.cases))
+    for tcr in ts.cases:
+        print("  Test case report: ", tcr.name)
+        print("  has #steps: ", len(tcr.steps))
+        for step in tcr.steps:
+            print("   step: ", step.name)
+            print("   has #queries: ", len(step.queries)) if step.get(
+                "queries"
+            ) is not None else print("   has #queries: 0")
+            for q in step.queries if step.get("queries") is not None else []:
+                print(f"    {q.response.elapsed_s} - {q.request.url}")
+
+
+def main():
+    """
+    Print some infos about a report's content.
+
+    Usage: python report_analyzer.py <path-to-report.json> <test_scenario_name>
+    Eg: python report_analyzer.py output/report_netrid_v22a.json "ASTM NetRID nominal behavior"
+    """
+    if len(sys.argv) < 2:
+        print(
+            "Usage: python report_analyzer.py <path-to-report.json> <test_scenario_name>"
+        )
+        return 1
+
+    r = parse_report(sys.argv[1])
+
+    for a in r.report.test_suite.actions:
+        print("Types of actions (test_suite, test_scenario, action_generator): ")
+        print(a._get_applicable_report())
+
+    suite_reports = {
+        r.test_suite.name: r.test_suite
+        for r in r.report.test_suite.actions
+        if r.get("test_suite") is not None
+    }
+    scenario_reports = {
+        r.test_scenario.name: r.test_scenario
+        for r in r.report.test_suite.actions
+        if r.get("test_scenario") is not None
+    }
+
+    print("Available suite reports: ", suite_reports.keys())
+    print("Available scenario reports: ", scenario_reports.keys())
+
+    if len(sys.argv) > 2:
+        look_at_scenario(scenario_reports[sys.argv[2]])
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())