-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[uss_qualifier/utm] Add AggregateChecks scenario with SCD0075 check (#…
…296)
- Loading branch information
Showing
11 changed files
with
204 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # ASTM F3548 UTM aggregate checks test scenario | ||
|
|
||
| ## Overview | ||
| In this special scenario, the report of previously executed ASTM F3548 UTM scenario(s) are evaluated for the | ||
| performances of the queries made during their execution. | ||
|
|
||
| ## Resources | ||
|
|
||
| ### report_resource | ||
| The report to evaluate. This resource is automatically injected by the test framework. | ||
|
|
||
| ### flight_planners | ||
| The flight planners subject to evaluation. | ||
|
|
||
| ## Performance of SCD requests to USS test case | ||
|
|
||
| ### Performance of successful operational intent details requests test step | ||
|
|
||
| In this step, all successful requests for operational intent details made to the USSs that are part of the flight | ||
| planners provided as resource are used to determine and evaluate the 95th percentile of the requests durations. | ||
|
|
||
| #### Operational intent details requests take no more than [MaxRespondToOIDetailsRequest] second 95% of the time check | ||
|
|
||
| If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second), | ||
| this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**. |
123 changes: 123 additions & 0 deletions
123
monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| from typing import List, Dict | ||
|
|
||
| from monitoring.monitorlib import fetch | ||
| from monitoring.monitorlib.fetch import evaluation, QueryType | ||
| from monitoring.uss_qualifier.common_data_definitions import Severity | ||
| from monitoring.uss_qualifier.configurations.configuration import ParticipantID | ||
| from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource | ||
| from monitoring.uss_qualifier.scenarios.interuss.evaluation_scenario import ( | ||
| ReportEvaluationScenario, | ||
| ) | ||
|
|
||
| from uas_standards.astm.f3548.v21 import constants | ||
|
|
||
| from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource | ||
|
|
||
|
|
||
| from monitoring.uss_qualifier.scenarios.scenario import TestScenario | ||
|
|
||
|
|
||
| class AggregateChecks(TestScenario, ReportEvaluationScenario): | ||
|
|
||
| _queries: List[fetch.Query] | ||
| _attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {} | ||
|
|
||
| def __init__( | ||
| self, | ||
| report_resource: TestSuiteReportResource, | ||
| flight_planners: FlightPlannersResource, | ||
| ): | ||
| super().__init__(report_resource) | ||
| self.flight_planners = flight_planners | ||
| self._queries = self.report.queries() | ||
|
|
||
| # collect and classify queries by participant, only participants part of flight_planners are considered | ||
| self._attributed_queries = { | ||
| flight_planner.flight_planner.participant_id: dict() | ||
| for flight_planner in self.flight_planners.flight_planners | ||
| } | ||
| for query in self._queries: | ||
| if not query.has_field_with_value( | ||
| "participant_id" | ||
| ) or not query.has_field_with_value("query_type"): | ||
| continue | ||
|
|
||
| if query.participant_id in self._attributed_queries: | ||
| if ( | ||
| query.query_type | ||
| not in self._attributed_queries[query.participant_id] | ||
| ): | ||
| self._attributed_queries[query.participant_id][ | ||
| query.query_type | ||
| ] = list() | ||
| self._attributed_queries[query.participant_id][query.query_type].append( | ||
| query | ||
| ) | ||
|
|
||
| def run(self): | ||
| self.begin_test_scenario() | ||
|
|
||
| self.record_note("all_queries", f"{len(self._queries)}") | ||
| for participant, queries_by_type in self._attributed_queries.items(): | ||
| self.record_note( | ||
| f"{participant}/attributed_queries", | ||
| ", ".join( | ||
| [ | ||
| f"{query_type}: {len(queries)}" | ||
| for query_type, queries in queries_by_type.items() | ||
| ] | ||
| ), | ||
| ) | ||
|
|
||
| self.begin_test_case("Performance of SCD requests to USS") | ||
| self.begin_test_step( | ||
| "Performance of successful operational intent details requests" | ||
| ) | ||
|
|
||
| self._op_intent_details_step() | ||
|
|
||
| self.end_test_step() | ||
| self.end_test_case() | ||
|
|
||
| self.end_test_scenario() | ||
|
|
||
| def _op_intent_details_step(self): | ||
| for participant, queries_by_type in self._attributed_queries.items(): | ||
| if ( | ||
| QueryType.F3548v21USSGetOperationalIntentDetails not in queries_by_type | ||
| or len( | ||
| queries_by_type[QueryType.F3548v21USSGetOperationalIntentDetails] | ||
| ) | ||
| == 0 | ||
| ): | ||
| self.record_note( | ||
| f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}", | ||
| "skipped check: no relevant queries", | ||
| ) | ||
| continue | ||
|
|
||
| queries = [ | ||
| query | ||
| for query in queries_by_type[ | ||
| QueryType.F3548v21USSGetOperationalIntentDetails | ||
| ] | ||
| if query.response.status_code == 200 | ||
| ] | ||
| durations = [query.response.elapsed_s for query in queries] | ||
| [p95] = evaluation.compute_percentiles(durations, [95]) | ||
| with self.check( | ||
| "Operational intent details requests take no more than [MaxRespondToOIDetailsRequest] second 95% of the time", | ||
| [participant], | ||
| ) as check: | ||
| if p95 > constants.MaxRespondToOIDetailsRequestSeconds: | ||
| check.record_failed( | ||
| summary=f"95th percentile of durations for operational intent details requests to USS is higher than threshold", | ||
| severity=Severity.Medium, | ||
| participants=[participant], | ||
| details=f"threshold: {constants.MaxRespondToOIDetailsRequestSeconds}s, 95th percentile: {p95}s", | ||
| ) | ||
|
|
||
| self.record_note( | ||
| f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}", | ||
| f"checked performances on {len(durations)} queries, 95th percentile: {p95}s", | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters