NET0260-a evaluate latency of calls to SP's /flights endpoint for dis…

…playing flights in a given area

monitoring/monitorlib/rid.py

@@ -164,6 +164,24 @@ def dp_data_resp_percentile99_s(self) -> float:
         else:
             raise ValueError("Unsupported RID version '{}'".format(self))
 
+    @property
+    def sp_data_resp_percentile95_s(self) -> float:
+        if self == RIDVersion.f3411_19:
+            return v19.constants.NetSpDataResponseTime95thPercentileSeconds
+        elif self == RIDVersion.f3411_22a:
+            return v22a.constants.NetSpDataResponseTime95thPercentileSeconds
+        else:
+            raise ValueError("Unsupported RID version '{}'".format(self))
+
+    @property
+    def sp_data_resp_percentile99_s(self) -> float:
+        if self == RIDVersion.f3411_19:
+            return v19.constants.NetSpDataResponseTime99thPercentileSeconds
+        elif self == RIDVersion.f3411_22a:
+            return v22a.constants.NetSpDataResponseTime99thPercentileSeconds
+        else:
+            raise ValueError("Unsupported RID version '{}'".format(self))
+
     def flights_url_of(self, base_url: str) -> str:
         if self == RIDVersion.f3411_19:
             return base_url

monitoring/uss_qualifier/configurations/dev/library/environment.yaml

@@ -17,6 +17,7 @@ net_rid:
             service_providers:
                 - participant_id: uss1
                   injection_base_url: http://host.docker.internal:8071/ridsp/injection
+                  uss_base_urls: [ http://host.docker.internal:8071 ]
     netrid_service_providers_v22a:
         $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json
         resource_type: resources.netrid.NetRIDServiceProviders
@@ -26,6 +27,7 @@ net_rid:
             service_providers:
                 - participant_id: uss1
                   injection_base_url: http://host.docker.internal:8081/ridsp/injection
+                  uss_base_urls: [ http://host.docker.internal:8081 ]
     netrid_observers_v19:
         $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json
         resource_type: resources.netrid.NetRIDObserversResource

monitoring/uss_qualifier/resources/netrid/observers.py

@@ -12,6 +12,18 @@
 from monitoring.uss_qualifier.resources.communications import AuthAdapterResource
 
 
+class ObserverConfiguration(ImplicitDict):
+    participant_id: str
+    """Participant ID of the observer providing a view of RID data in the system"""
+
+    observation_base_url: str
+    """Base URL for the observer's implementation of the interfaces/automated-testing/rid/observation.yaml API"""
+
+
+class NetRIDObserversSpecification(ImplicitDict):
+    observers: List[ObserverConfiguration]
+
+
 class RIDSystemObserver(object):
     participant_id: str
     base_url: str
@@ -74,18 +86,6 @@ def observe_flight_details(
         return result, query
 
 
-class ObserverConfiguration(ImplicitDict):
-    participant_id: str
-    """Participant ID of the observer providing a view of RID data in the system"""
-
-    observation_base_url: str
-    """Base URL for the observer's implementation of the interfaces/automated-testing/rid/observation.yaml API"""
-
-
-class NetRIDObserversSpecification(ImplicitDict):
-    observers: List[ObserverConfiguration]
-
-
 class NetRIDObserversResource(Resource[NetRIDObserversSpecification]):
     observers: List[RIDSystemObserver]
 
@@ -96,7 +96,9 @@ def __init__(
     ):
         self.observers = [
             RIDSystemObserver(
-                o.participant_id, o.observation_base_url, auth_adapter.adapter
+                o.participant_id,
+                o.observation_base_url,
+                auth_adapter.adapter,
             )
             for o in specification.observers
         ]

monitoring/uss_qualifier/resources/netrid/service_providers.py

@@ -20,6 +20,10 @@ class ServiceProviderConfiguration(ImplicitDict):
     injection_base_url: str
     """Base URL for the Service Provider's implementation of the interfaces/automated-testing/rid/injection.yaml API"""
 
+    uss_base_urls: List[str]
+    """Base URLS where to reach this particular USS. We allow multiple ones as nothing prevents a
+    USS to advertise multiple different endpoints where to be contacted."""
+
     def __init__(self, *args, **kwargs):
         super().__init__(**kwargs)
         try:
@@ -29,6 +33,14 @@ def __init__(self, *args, **kwargs):
                 "ServiceProviderConfiguration.injection_base_url must be a URL"
             )
 
+        for uss_base_url in self.uss_base_urls:
+            try:
+                urlparse(uss_base_url)
+            except ValueError:
+                raise ValueError(
+                    f"ServiceProviderConfiguration.uss_base_urls must contain valid URLs. Was: {uss_base_url}",
+                )
+
 
 class NetRIDServiceProvidersSpecification(ImplicitDict):
     service_providers: List[ServiceProviderConfiguration]
@@ -38,16 +50,19 @@ class NetRIDServiceProvider(object):
     participant_id: str
     base_url: str
     client: infrastructure.UTMClientSession
+    original_configuration: ServiceProviderConfiguration
 
     def __init__(
         self,
         participant_id: str,
         base_url: str,
         auth_adapter: infrastructure.AuthAdapter,
+        original_configuration: ServiceProviderConfiguration,
     ):
         self.participant_id = participant_id
         self.base_url = base_url
         self.client = infrastructure.UTMClientSession(base_url, auth_adapter)
+        self.original_configuration = original_configuration
 
     def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Query:
         return fetch.query_and_describe(
@@ -77,7 +92,7 @@ def __init__(
     ):
         self.service_providers = [
             NetRIDServiceProvider(
-                s.participant_id, s.injection_base_url, auth_adapter.adapter
+                s.participant_id, s.injection_base_url, auth_adapter.adapter, s
             )
             for s in specification.service_providers
         ]

monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py

@@ -1,6 +1,8 @@
 import re
 from typing import List, Dict
 
+import urllib.parse
+
 from monitoring.monitorlib import fetch
 from monitoring.monitorlib.fetch import evaluation
 from monitoring.monitorlib.rid import RIDVersion
@@ -41,13 +43,23 @@ def __init__(
         self._service_providers = service_providers.service_providers
         self._observers = observers.observers
 
-        # identify SPs and observers by their base URL
-        self._participants_by_base_url.update(
-            {sp.base_url: sp.participant_id for sp in self._service_providers}
-        )
-        self._participants_by_base_url.update(
-            {dp.base_url: dp.participant_id for dp in self._observers}
-        )
+        # identify SPs and observers by the base urls specified in their configuration
+        for sp in self._service_providers:
+            for base_url in sp.original_configuration.uss_base_urls:
+                prev_mapping = self._participants_by_base_url.get(base_url)
+                if prev_mapping is not None and prev_mapping != sp.participant_id:
+                    raise ValueError(
+                        f"Invalid configuration detected: same uss base url is defined for mutiple participants. Url: {base_url}, participants: {prev_mapping}, {sp.participant_id}"
+                    )
+                self._participants_by_base_url[base_url] = sp.participant_id
+
+        for obs in self._observers:
+            prev_mapping = self._participants_by_base_url.get(obs.base_url)
+            if prev_mapping is not None and prev_mapping != obs.participant_id:
+                raise ValueError(
+                    f"Invalid configuration detected: same uss base url is defined for mutiple participants. Url: {obs.base_url}, participants: {prev_mapping}, {obs.participant_id}"
+                )
+            self._participants_by_base_url[base_url] = obs.participant_id
 
         # collect and classify queries by participant
         self._queries_by_participant = {
@@ -61,19 +73,72 @@ def __init__(
                     break
 
     def run(self):
+
         self.begin_test_scenario()
         self.record_note("participants", str(self._participants_by_base_url))
         self.record_note("nb_queries", str(len(self._queries)))
 
+        # DP performance
         self.begin_test_case("Performance of Display Providers requests")
-
         self.begin_test_step("Performance of /display_data requests")
+
         self._dp_display_data_times_step()
+
         self.end_test_step()
+        self.end_test_case()
+
+        # SP performance
+        self.begin_test_case("Performance of Service Providers requests")
+        self.begin_test_step("Performance of /flights?view requests")
 
+        self._sp_flights_area_times_step()
+
+        self.end_test_step()
         self.end_test_case()
+
         self.end_test_scenario()
 
+    def _sp_flights_area_times_step(self):
+        pattern = re.compile(r"v2/uss/flights\?view=")
+        for participant, all_queries in self._queries_by_participant.items():
+            # identify successful flights queries
+            relevant_queries: List[fetch.Query] = list()
+            for query in all_queries:
+                match = pattern.search(query.request.url)
+                if match is not None and query.status_code == 200:
+                    relevant_queries.append(query)
+
+            if len(relevant_queries) == 0:
+                # this may be a display provider
+                self.record_note(
+                    f"{participant}/flights", "skipped check: no relevant queries"
+                )
+
+                continue
+
+            # Collect query durations
+            durations = [query.response.elapsed_s for query in relevant_queries]
+            (p95, p99) = evaluation.compute_percentiles(durations, [95, 99])
+
+            with self.check(
+                "Performance for replies to requested flights in an area", [participant]
+            ) as check:
+                if p95 > self._rid_version.sp_data_resp_percentile95_s:
+                    check.record_failed(
+                        f"95th percentile of /flights?view requests is {p95} s, "
+                        f"expected less than {self._rid_version.sp_data_resp_percentile95_s} s"
+                    )
+                if p99 > self._rid_version.sp_data_resp_percentile99_s:
+                    check.record_failed(
+                        f"99th percentile of /flights?view requests is {p99} s, "
+                        f"expected less than {self._rid_version.sp_data_resp_percentile99_s} s"
+                    )
+
+            self.record_note(
+                f"{participant}/flights",
+                f"percentiles on {len(relevant_queries)} relevant queries: 95th: {p95}; 99th: {p99}",
+            )
+
     def _dp_display_data_times_step(self):
         """
         :return: the query durations of respectively the initial queries and the subsequent ones
@@ -119,7 +184,7 @@ def _dp_display_data_times_step(self):
                         summary=f"95th percentile of durations for initial DP display_data queries is higher than threshold",
                         severity=Severity.Medium,
                         participants=[participant],
-                        details=f"threshold: {self._rid_version.dp_init_resp_percentile95_s}, 95th percentile: {init_95th}",
+                        details=f"threshold: {self._rid_version.sp_init_resp_percentile95_s}, 95th percentile: {init_95th}",
                     )
                 if init_99th > self._rid_version.dp_init_resp_percentile99_s:
                     check.record_failed(

monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md

@@ -4,7 +4,6 @@
 In this special scenario, the report of previously executed ASTM F3411-19 NetRID scenario(s) are evaluated for the
 performances of the queries made during their execution.
 
-
 ## Resources
 
 ### report_resource
@@ -16,7 +15,6 @@ The service providers to evaluate in the report.
 ### observers
 The observers to evaluate in the report.
 
-
 ## Performance of Display Providers requests test case
 
 ### Performance of /display_data requests test step
@@ -34,3 +32,13 @@ of the durations for the initial display data queries do not exceed the respecti
 **[astm.f3411.v19.NET0440](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles
 of the durations for the subsequent display data queries do not exceed the respectives thresholds
 `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`.
+
+## Performance of Service Providers requests test case
+
+### Performance of /flights?view requests test step
+
+#### Performance for replies to requested flights in an area check
+
+**[astm.f3411.v19.NET0260-a](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles
+of the durations for the replies to requested flights in an area do not exceed the respective thresholds
+`NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds).

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md

@@ -16,7 +16,6 @@ The service providers to evaluate in the report.
 ### observers
 The observers to evaluate in the report.
 
-
 ## Performance of Display Providers requests test case
 
 ### Performance of /display_data requests test step
@@ -34,3 +33,13 @@ of the durations for the initial display data queries do not exceed the respecti
 **[astm.f3411.v22a.NET0440](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles
 of the durations for the subsequent display data queries do not exceed the respectives thresholds
 `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`.
+
+## Performance of Service Providers requests test case
+
+### Performance of /flights?view requests test step
+
+#### Performance for replies to requested flights in an area check
+
+**[astm.f3411.v22a.NET0260-a](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles
+of the durations for the replies to requested flights in an area do not exceed the respective thresholds
+`NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds).

monitoring/uss_qualifier/scripts/report_analyzer.py

@@ -0,0 +1,77 @@
+import sys
+import json
+
+from implicitdict import ImplicitDict
+
+from monitoring.uss_qualifier.reports.report import (
+    TestRunReport,
+    TestSuiteReport,
+    TestScenarioReport,
+)
+
+
+def parse_report(path: str) -> TestRunReport:
+    with open(path, "r") as f:
+        report = json.load(f)
+        return ImplicitDict.parse(report, TestRunReport)
+
+
+def look_at_test_suite(ts: TestSuiteReport):
+    print("test-suite: " + ts.name)
+
+
+def look_at_scenario(ts: TestScenarioReport):
+    print("Looking at test scenario: ", ts.name)
+    print("Has #cases: ", len(ts.cases))
+    for tcr in ts.cases:
+        print("  Test case report: ", tcr.name)
+        print("  has #steps: ", len(tcr.steps))
+        for step in tcr.steps:
+            print("   step: ", step.name)
+            print("   has #queries: ", len(step.queries)) if step.get(
+                "queries"
+            ) is not None else print("   has #queries: 0")
+            for q in step.queries if step.get("queries") is not None else []:
+                print(f"    {q.response.elapsed_s} - {q.request.url}")
+
+
+def main():
+    """
+    Print some infos about a report's content.
+
+    Usage: python report_analyzer.py <path-to-report.json> <test_scenario_name>
+    Eg: python report_analyzer.py output/report_netrid_v22a.json "ASTM NetRID nominal behavior"
+    """
+    if len(sys.argv) < 2:
+        print(
+            "Usage: python report_analyzer.py <path-to-report.json> <test_scenario_name>"
+        )
+        return 1
+
+    r = parse_report(sys.argv[1])
+
+    for a in r.report.test_suite.actions:
+        print("Types of actions (test_suite, test_scenario, action_generator): ")
+        print(a._get_applicable_report())
+
+    suite_reports = {
+        r.test_suite.name: r.test_suite
+        for r in r.report.test_suite.actions
+        if r.get("test_suite") is not None
+    }
+    scenario_reports = {
+        r.test_scenario.name: r.test_scenario
+        for r in r.report.test_suite.actions
+        if r.get("test_scenario") is not None
+    }
+
+    print("Available suite reports: ", suite_reports.keys())
+    print("Available scenario reports: ", scenario_reports.keys())
+
+    if len(sys.argv) > 2:
+        look_at_scenario(scenario_reports[sys.argv[2]])
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())