From 47d942f4ac6bd53b39da73bc456d18a7ac1e0858 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Thu, 10 Aug 2023 12:11:31 +0200 Subject: [PATCH] NET0260-a evaluate latency of calls to SP's /flights endpoint for displaying flights in a given area --- monitoring/monitorlib/rid.py | 18 ++++ .../dev/library/environment.yaml | 4 + .../resources/netrid/observers.py | 33 +++++--- .../resources/netrid/service_providers.py | 17 +++- .../astm/netrid/common/aggregate_checks.py | 84 +++++++++++++++++-- .../astm/netrid/v19/aggregate_checks.md | 12 ++- .../astm/netrid/v22a/aggregate_checks.md | 11 ++- .../uss_qualifier/scripts/report_analyzer.py | 77 +++++++++++++++++ 8 files changed, 230 insertions(+), 26 deletions(-) create mode 100644 monitoring/uss_qualifier/scripts/report_analyzer.py diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index 3b43e39182..5354577549 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -164,6 +164,24 @@ def dp_data_resp_percentile99_s(self) -> float: else: raise ValueError("Unsupported RID version '{}'".format(self)) + @property + def sp_data_resp_percentile95_s(self) -> float: + if self == RIDVersion.f3411_19: + return v19.constants.NetSpDataResponseTime95thPercentileSeconds + elif self == RIDVersion.f3411_22a: + return v22a.constants.NetSpDataResponseTime95thPercentileSeconds + else: + raise ValueError("Unsupported RID version '{}'".format(self)) + + @property + def sp_data_resp_percentile99_s(self) -> float: + if self == RIDVersion.f3411_19: + return v19.constants.NetSpDataResponseTime99thPercentileSeconds + elif self == RIDVersion.f3411_22a: + return v22a.constants.NetSpDataResponseTime99thPercentileSeconds + else: + raise ValueError("Unsupported RID version '{}'".format(self)) + def flights_url_of(self, base_url: str) -> str: if self == RIDVersion.f3411_19: return base_url diff --git a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml index d230912044..ee8099154f 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml @@ -17,6 +17,7 @@ net_rid: service_providers: - participant_id: uss1 injection_base_url: http://host.docker.internal:8071/ridsp/injection + uss_base_urls: [ http://host.docker.internal:8071 ] netrid_service_providers_v22a: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.NetRIDServiceProviders @@ -26,6 +27,7 @@ net_rid: service_providers: - participant_id: uss1 injection_base_url: http://host.docker.internal:8081/ridsp/injection + uss_base_urls: [ http://host.docker.internal:8081 ] netrid_observers_v19: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.NetRIDObserversResource @@ -35,6 +37,7 @@ net_rid: observers: - participant_id: uss2 observation_base_url: http://host.docker.internal:8073/riddp/observation + uss_base_urls: [ http://host.docker.internal:8073 ] netrid_observers_v22a: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.NetRIDObserversResource @@ -44,6 +47,7 @@ net_rid: observers: - participant_id: uss2 observation_base_url: http://host.docker.internal:8083/riddp/observation + uss_base_urls: [ http://host.docker.internal:8083 ] netrid_dss_instances_v19: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.astm.f3411.DSSInstancesResource diff --git a/monitoring/uss_qualifier/resources/netrid/observers.py b/monitoring/uss_qualifier/resources/netrid/observers.py index 70a8737d39..3862e3f607 100644 --- a/monitoring/uss_qualifier/resources/netrid/observers.py +++ b/monitoring/uss_qualifier/resources/netrid/observers.py @@ -12,20 +12,39 @@ from monitoring.uss_qualifier.resources.communications import AuthAdapterResource +class ObserverConfiguration(ImplicitDict): + participant_id: str + """Participant ID of the observer providing a view of RID data in the system""" + + observation_base_url: str + """Base URL for the observer's implementation of the interfaces/automated-testing/rid/observation.yaml API""" + + uss_base_urls: List[str] + """Base URLS where to reach this particular USS. We allow multiple ones as nothing prevents a + USS to advertise multiple different endpoints where to be contacted.""" + + +class NetRIDObserversSpecification(ImplicitDict): + observers: List[ObserverConfiguration] + + class RIDSystemObserver(object): participant_id: str base_url: str session: infrastructure.UTMClientSession + original_configuration: NetRIDObserversSpecification def __init__( self, participant_id: str, base_url: str, auth_adapter: infrastructure.AuthAdapter, + original_configuration: NetRIDObserversSpecification, ): self.session = UTMClientSession(base_url, auth_adapter) self.participant_id = participant_id self.base_url = base_url + self.original_configuration = original_configuration # TODO: Change observation API to use an InterUSS scope rather than re-using an ASTM scope self.rid_version = RIDVersion.f3411_19 @@ -74,18 +93,6 @@ def observe_flight_details( return result, query -class ObserverConfiguration(ImplicitDict): - participant_id: str - """Participant ID of the observer providing a view of RID data in the system""" - - observation_base_url: str - """Base URL for the observer's implementation of the interfaces/automated-testing/rid/observation.yaml API""" - - -class NetRIDObserversSpecification(ImplicitDict): - observers: List[ObserverConfiguration] - - class NetRIDObserversResource(Resource[NetRIDObserversSpecification]): observers: List[RIDSystemObserver] @@ -96,7 +103,7 @@ def __init__( ): self.observers = [ RIDSystemObserver( - o.participant_id, o.observation_base_url, auth_adapter.adapter + o.participant_id, o.observation_base_url, auth_adapter.adapter, o ) for o in specification.observers ] diff --git a/monitoring/uss_qualifier/resources/netrid/service_providers.py b/monitoring/uss_qualifier/resources/netrid/service_providers.py index f4ec02a91d..8ba68a481b 100644 --- a/monitoring/uss_qualifier/resources/netrid/service_providers.py +++ b/monitoring/uss_qualifier/resources/netrid/service_providers.py @@ -20,6 +20,10 @@ class ServiceProviderConfiguration(ImplicitDict): injection_base_url: str """Base URL for the Service Provider's implementation of the interfaces/automated-testing/rid/injection.yaml API""" + uss_base_urls: List[str] + """Base URLS where to reach this particular USS. We allow multiple ones as nothing prevents a + USS to advertise multiple different endpoints where to be contacted.""" + def __init__(self, *args, **kwargs): super().__init__(**kwargs) try: @@ -29,6 +33,14 @@ def __init__(self, *args, **kwargs): "ServiceProviderConfiguration.injection_base_url must be a URL" ) + for uss_base_url in self.uss_base_urls: + try: + urlparse(uss_base_url) + except ValueError: + raise ValueError( + f"ServiceProviderConfiguration.uss_base_urls must contain valid URLs. Was: {uss_base_url}", + ) + class NetRIDServiceProvidersSpecification(ImplicitDict): service_providers: List[ServiceProviderConfiguration] @@ -38,16 +50,19 @@ class NetRIDServiceProvider(object): participant_id: str base_url: str client: infrastructure.UTMClientSession + original_configuration: ServiceProviderConfiguration def __init__( self, participant_id: str, base_url: str, auth_adapter: infrastructure.AuthAdapter, + original_configuration: ServiceProviderConfiguration, ): self.participant_id = participant_id self.base_url = base_url self.client = infrastructure.UTMClientSession(base_url, auth_adapter) + self.original_configuration = original_configuration def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Query: return fetch.query_and_describe( @@ -77,7 +92,7 @@ def __init__( ): self.service_providers = [ NetRIDServiceProvider( - s.participant_id, s.injection_base_url, auth_adapter.adapter + s.participant_id, s.injection_base_url, auth_adapter.adapter, s ) for s in specification.service_providers ] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index 4c93634652..70fc503e0b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -1,6 +1,8 @@ import re from typing import List, Dict +import urllib.parse + from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import evaluation from monitoring.monitorlib.rid import RIDVersion @@ -41,13 +43,24 @@ def __init__( self._service_providers = service_providers.service_providers self._observers = observers.observers - # identify SPs and observers by their base URL - self._participants_by_base_url.update( - {sp.base_url: sp.participant_id for sp in self._service_providers} - ) - self._participants_by_base_url.update( - {dp.base_url: dp.participant_id for dp in self._observers} - ) + # identify SPs and observers by their hostnames + ports + for sp in self._service_providers: + for base_url in sp.original_configuration.uss_base_urls: + prev_mapping = self._participants_by_base_url.get(base_url) + if prev_mapping is not None and prev_mapping != sp.participant_id: + raise ValueError( + f"Invalid configuration detected: same uss base url is defined for mutiple participants. Url: {base_url}, participants: {prev_mapping}, {sp.participant_id}" + ) + self._participants_by_base_url[base_url] = sp.participant_id + + for obs in self._observers: + for base_url in obs.original_configuration.uss_base_urls: + prev_mapping = self._participants_by_base_url.get(base_url) + if prev_mapping is not None and prev_mapping != obs.participant_id: + raise ValueError( + f"Invalid configuration detected: same uss base url is defined for mutiple participants. Url: {base_url}, participants: {prev_mapping}, {obs.participant_id}" + ) + self._participants_by_base_url[base_url] = obs.participant_id # collect and classify queries by participant self._queries_by_participant = { @@ -61,19 +74,72 @@ def __init__( break def run(self): + self.begin_test_scenario() self.record_note("participants", str(self._participants_by_base_url)) self.record_note("nb_queries", str(len(self._queries))) + # DP performance self.begin_test_case("Performance of Display Providers requests") - self.begin_test_step("Performance of /display_data requests") + self._dp_display_data_times_step() + self.end_test_step() + self.end_test_case() + + # SP performance + self.begin_test_case("Performance of Service Providers requests") + self.begin_test_step("Performance of /flights?view requests") + self._sp_flights_area_times_step() + + self.end_test_step() self.end_test_case() + self.end_test_scenario() + def _sp_flights_area_times_step(self): + pattern = re.compile(r"/flights\?view=") + for participant, all_queries in self._queries_by_participant.items(): + # identify successful flights queries + relevant_queries: List[fetch.Query] = list() + for query in all_queries: + match = pattern.search(query.request.url) + if match is not None and query.status_code == 200: + relevant_queries.append(query) + + if len(relevant_queries) == 0: + # this may be a display provider + self.record_note( + f"{participant}/flights", "skipped check: no relevant queries" + ) + + continue + + # Collect query durations + durations = [query.response.elapsed_s for query in relevant_queries] + (p95, p99) = evaluation.compute_percentiles(durations, [95, 99]) + + with self.check( + "Performance for replies to requested flights in an area", [participant] + ) as check: + if p95 > self._rid_version.sp_data_resp_percentile95_s: + check.record_failed( + f"95th percentile of /flights?view requests is {p95} s, " + f"expected less than {self._rid_version.sp_data_resp_percentile95_s} s" + ) + if p99 > self._rid_version.sp_data_resp_percentile99_s: + check.record_failed( + f"99th percentile of /flights?view requests is {p99} s, " + f"expected less than {self._rid_version.sp_data_resp_percentile99_s} s" + ) + + self.record_note( + f"{participant}/flights", + f"percentiles on {len(relevant_queries)} relevant queries: 95th: {p95}; 99th: {p99}", + ) + def _dp_display_data_times_step(self): """ :return: the query durations of respectively the initial queries and the subsequent ones @@ -119,7 +185,7 @@ def _dp_display_data_times_step(self): summary=f"95th percentile of durations for initial DP display_data queries is higher than threshold", severity=Severity.Medium, participants=[participant], - details=f"threshold: {self._rid_version.dp_init_resp_percentile95_s}, 95th percentile: {init_95th}", + details=f"threshold: {self._rid_version.sp_init_resp_percentile95_s}, 95th percentile: {init_95th}", ) if init_99th > self._rid_version.dp_init_resp_percentile99_s: check.record_failed( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index dead5ac209..8dcbdfead6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -4,7 +4,6 @@ In this special scenario, the report of previously executed ASTM F3411-19 NetRID scenario(s) are evaluated for the performances of the queries made during their execution. - ## Resources ### report_resource @@ -16,7 +15,6 @@ The service providers to evaluate in the report. ### observers The observers to evaluate in the report. - ## Performance of Display Providers requests test case ### Performance of /display_data requests test step @@ -34,3 +32,13 @@ of the durations for the initial display data queries do not exceed the respecti **[astm.f3411.v19.NET0440](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles of the durations for the subsequent display data queries do not exceed the respectives thresholds `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`. + +## Performance of Service Providers requests test case + +### Performance of /flights?view requests test step + +#### Performance for replies to requested flights in an area check + +**[astm.f3411.v19.NET0260-a](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles +of the durations for the replies to requested flights in an area do not exceed the respective thresholds +`NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 279a470e3e..17760ea7eb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -16,7 +16,6 @@ The service providers to evaluate in the report. ### observers The observers to evaluate in the report. - ## Performance of Display Providers requests test case ### Performance of /display_data requests test step @@ -34,3 +33,13 @@ of the durations for the initial display data queries do not exceed the respecti **[astm.f3411.v22a.NET0440](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles of the durations for the subsequent display data queries do not exceed the respectives thresholds `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`. + +## Performance of Service Providers requests test case + +### Performance of /flights?view requests test step + +#### Performance for replies to requested flights in an area check + +**[astm.f3411.v22a.NET0260-a](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles +of the durations for the replies to requested flights in an area do not exceed the respective thresholds +`NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). diff --git a/monitoring/uss_qualifier/scripts/report_analyzer.py b/monitoring/uss_qualifier/scripts/report_analyzer.py new file mode 100644 index 0000000000..605ce0ba16 --- /dev/null +++ b/monitoring/uss_qualifier/scripts/report_analyzer.py @@ -0,0 +1,77 @@ +import sys +import json + +from implicitdict import ImplicitDict + +from monitoring.uss_qualifier.reports.report import ( + TestRunReport, + TestSuiteReport, + TestScenarioReport, +) + + +def parse_report(path: str) -> TestRunReport: + with open(path, "r") as f: + report = json.load(f) + return ImplicitDict.parse(report, TestRunReport) + + +def look_at_test_suite(ts: TestSuiteReport): + print("test-suite: " + ts.name) + + +def look_at_scenario(ts: TestScenarioReport): + print("Looking at test scenario: ", ts.name) + print("Has #cases: ", len(ts.cases)) + for tcr in ts.cases: + print(" Test case report: ", tcr.name) + print(" has #steps: ", len(tcr.steps)) + for step in tcr.steps: + print(" step: ", step.name) + print(" has #queries: ", len(step.queries)) if step.get( + "queries" + ) is not None else print(" has #queries: 0") + for q in step.queries if step.get("queries") is not None else []: + print(f" {q.response.elapsed_s} - {q.request.url}") + + +def main(): + """ + Print some infos about a report's content. + + Usage: python report_analyzer.py + Eg: python report_analyzer.py output/report_netrid_v22a.json "ASTM NetRID nominal behavior" + """ + if len(sys.argv) < 2: + print( + "Usage: python report_analyzer.py " + ) + return 1 + + r = parse_report(sys.argv[1]) + + for a in r.report.test_suite.actions: + print("Types of actions (test_suite, test_scenario, action_generator): ") + print(a._get_applicable_report()) + + suite_reports = { + r.test_suite.name: r.test_suite + for r in r.report.test_suite.actions + if r.get("test_suite") is not None + } + scenario_reports = { + r.test_scenario.name: r.test_scenario + for r in r.report.test_suite.actions + if r.get("test_scenario") is not None + } + + print("Available suite reports: ", suite_reports.keys()) + print("Available scenario reports: ", scenario_reports.keys()) + + if len(sys.argv) > 2: + look_at_scenario(scenario_reports[sys.argv[2]]) + return 0 + + +if __name__ == "__main__": + sys.exit(main())