From 9ca90318d5dc9cb90d138282dd56e443093689a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Sat, 28 Oct 2023 22:03:41 +0200 Subject: [PATCH] [uss_qualifier/utm] Add AggregateChecks scenario with SCD0075 check (#296) --- .../resources/astm/f3548/v21/dss.py | 15 ++- .../scenarios/astm/utm/__init__.py | 1 + .../scenarios/astm/utm/aggregate_checks.md | 25 ++++ .../scenarios/astm/utm/aggregate_checks.py | 123 ++++++++++++++++++ .../scenarios/astm/utm/test_steps.py | 4 +- .../uss_qualifier/scenarios/scenario.py | 12 +- .../uss_qualifier/suites/astm/utm/f3548_21.md | 8 +- .../suites/astm/utm/f3548_21.yaml | 4 + .../suites/faa/uft/message_signing.md | 7 +- .../suites/uspace/flight_auth.md | 7 +- .../suites/uspace/required_services.md | 7 +- 11 files changed, 204 insertions(+), 9 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index b416ad3121..dca6696820 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -5,6 +5,7 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import infrastructure, fetch +from monitoring.monitorlib.fetch import QueryType from monitoring.monitorlib.scd import SCOPE_SC from monitoring.uss_qualifier.resources.resource import Resource from monitoring.uss_qualifier.resources.communications import AuthAdapterResource @@ -64,9 +65,10 @@ def find_op_intent( self.client, "POST", url, + QueryType.F3548v21DSSQueryOperationalIntentReferences, + self.participant_id, scope=SCOPE_SC, json=req, - participant_id=self.participant_id, ) if query.status_code != 200: result = None @@ -77,11 +79,18 @@ def find_op_intent( return result, query def get_full_op_intent( - self, op_intent_ref: OperationalIntentReference + self, + op_intent_ref: OperationalIntentReference, + uss_participant_id: Optional[str] = None, ) -> Tuple[OperationalIntent, fetch.Query]: url = f"{op_intent_ref.uss_base_url}/uss/v1/operational_intents/{op_intent_ref.id}" query = fetch.query_and_describe( - self.client, "GET", url, scope=SCOPE_SC, participant_id=self.participant_id + self.client, + "GET", + url, + QueryType.F3548v21USSGetOperationalIntentDetails, + uss_participant_id, + scope=SCOPE_SC, ) if query.status_code != 200: result = None diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py index 9317644cb6..8a5637e620 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py @@ -6,3 +6,4 @@ ConflictEqualPriorityNotPermitted, ) from .dss_interoperability import DSSInteroperability +from .aggregate_checks import AggregateChecks diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md new file mode 100644 index 0000000000..b43d4031d1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md @@ -0,0 +1,25 @@ +# ASTM F3548 UTM aggregate checks test scenario + +## Overview +In this special scenario, the report of previously executed ASTM F3548 UTM scenario(s) are evaluated for the +performances of the queries made during their execution. + +## Resources + +### report_resource +The report to evaluate. This resource is automatically injected by the test framework. + +### flight_planners +The flight planners subject to evaluation. + +## Performance of SCD requests to USS test case + +### Performance of successful operational intent details requests test step + +In this step, all successful requests for operational intent details made to the USSs that are part of the flight +planners provided as resource are used to determine and evaluate the 95th percentile of the requests durations. + +#### Operational intent details requests take no more than [MaxRespondToOIDetailsRequest] second 95% of the time check + +If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second), +this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py new file mode 100644 index 0000000000..218b2cbd54 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py @@ -0,0 +1,123 @@ +from typing import List, Dict + +from monitoring.monitorlib import fetch +from monitoring.monitorlib.fetch import evaluation, QueryType +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource +from monitoring.uss_qualifier.scenarios.interuss.evaluation_scenario import ( + ReportEvaluationScenario, +) + +from uas_standards.astm.f3548.v21 import constants + +from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource + + +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class AggregateChecks(TestScenario, ReportEvaluationScenario): + + _queries: List[fetch.Query] + _attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {} + + def __init__( + self, + report_resource: TestSuiteReportResource, + flight_planners: FlightPlannersResource, + ): + super().__init__(report_resource) + self.flight_planners = flight_planners + self._queries = self.report.queries() + + # collect and classify queries by participant, only participants part of flight_planners are considered + self._attributed_queries = { + flight_planner.flight_planner.participant_id: dict() + for flight_planner in self.flight_planners.flight_planners + } + for query in self._queries: + if not query.has_field_with_value( + "participant_id" + ) or not query.has_field_with_value("query_type"): + continue + + if query.participant_id in self._attributed_queries: + if ( + query.query_type + not in self._attributed_queries[query.participant_id] + ): + self._attributed_queries[query.participant_id][ + query.query_type + ] = list() + self._attributed_queries[query.participant_id][query.query_type].append( + query + ) + + def run(self): + self.begin_test_scenario() + + self.record_note("all_queries", f"{len(self._queries)}") + for participant, queries_by_type in self._attributed_queries.items(): + self.record_note( + f"{participant}/attributed_queries", + ", ".join( + [ + f"{query_type}: {len(queries)}" + for query_type, queries in queries_by_type.items() + ] + ), + ) + + self.begin_test_case("Performance of SCD requests to USS") + self.begin_test_step( + "Performance of successful operational intent details requests" + ) + + self._op_intent_details_step() + + self.end_test_step() + self.end_test_case() + + self.end_test_scenario() + + def _op_intent_details_step(self): + for participant, queries_by_type in self._attributed_queries.items(): + if ( + QueryType.F3548v21USSGetOperationalIntentDetails not in queries_by_type + or len( + queries_by_type[QueryType.F3548v21USSGetOperationalIntentDetails] + ) + == 0 + ): + self.record_note( + f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}", + "skipped check: no relevant queries", + ) + continue + + queries = [ + query + for query in queries_by_type[ + QueryType.F3548v21USSGetOperationalIntentDetails + ] + if query.response.status_code == 200 + ] + durations = [query.response.elapsed_s for query in queries] + [p95] = evaluation.compute_percentiles(durations, [95]) + with self.check( + "Operational intent details requests take no more than [MaxRespondToOIDetailsRequest] second 95% of the time", + [participant], + ) as check: + if p95 > constants.MaxRespondToOIDetailsRequestSeconds: + check.record_failed( + summary=f"95th percentile of durations for operational intent details requests to USS is higher than threshold", + severity=Severity.Medium, + participants=[participant], + details=f"threshold: {constants.MaxRespondToOIDetailsRequestSeconds}s, 95th percentile: {p95}s", + ) + + self.record_note( + f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}", + f"checked performances on {len(durations)} queries, 95th percentile: {p95}s", + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index 46bf3e9c60..4a823c912f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -218,7 +218,9 @@ def expect_shared( ) oi_ref = self._new_oi_ref - oi_full, oi_full_query = self._dss.get_full_op_intent(oi_ref) + oi_full, oi_full_query = self._dss.get_full_op_intent( + oi_ref, self._flight_planner.participant_id + ) self._scenario.record_query(oi_full_query) with self._scenario.check( "Operational intent details retrievable", diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 30db6314d6..a08028f000 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -342,8 +342,18 @@ def record_query(self, query: fetch.Query) -> None: if "queries" not in self._step_report: self._step_report.queries = [] self._step_report.queries.append(query) + participant = ( + "UNKNOWN" + if not query.has_field_with_value("participant_id") + else query.participant_id + ) + query_type = ( + "UNKNOWN" + if not query.has_field_with_value("query_type") + else query.query_type + ) logger.debug( - f"Queried {query.request['method']} {query.request['url']} -> {query.response.status_code} ({query.response.elapsed_s:.1f}s)" + f"Queried {query.request['method']} {query.request['url']} -> {query.response.status_code} ({query.response.elapsed_s:.1f}s); attributed participant {participant} and type {query_type}" ) def _get_check(self, name: str) -> TestCheckDocumentation: diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index d0baee191d..cdaff5b1c8 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -12,6 +12,7 @@ 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) 4. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) +5. Scenario: [ASTM F3548 UTM aggregate checks](../../../scenarios/astm/utm/aggregate_checks.md) ([`scenarios.astm.utm.AggregateChecks`](../../../scenarios/astm/utm/aggregate_checks.py)) ## [Checked requirements](../../README.md#checked-requirements) @@ -23,7 +24,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -108,6 +109,11 @@ Implemented Nominal planning: not permitted conflict with equal priority + + SCD0075 + Implemented + ASTM F3548 UTM aggregate checks + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index d2438492d8..41dbededa3 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -91,6 +91,10 @@ actions: - uss1 - uss2 on_failure: Continue +report_evaluation_scenario: + scenario_type: scenarios.astm.utm.AggregateChecks + resources: + flight_planners: flight_planners participant_verifiable_capabilities: - id: scd name: Strategic Conflict Detection diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 4296159689..c652b2f5fa 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -103,6 +103,11 @@ Implemented Nominal planning: not permitted conflict with equal priority + + SCD0075 + Implemented + ASTM F3548 UTM aggregate checks + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 659b1a805d..a8659576a2 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -103,6 +103,11 @@ Implemented Nominal planning: not permitted conflict with equal priority + + SCD0075 + Implemented + ASTM F3548 UTM aggregate checks + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 9a1b53a142..d72dfe8232 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -449,7 +449,7 @@ ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005 Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -534,6 +534,11 @@ Implemented Nominal planning: not permitted conflict with equal priority + + SCD0075 + Implemented + ASTM F3548 UTM aggregate checks + USS0005 Implemented