From bc3fb8baf826e369a3e9869b0263a991a5a25f08 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 25 Mar 2024 06:45:50 -0700 Subject: [PATCH] [uss_qualifier] Verify area is clear before nominal planning equal priority (#593) Verify area is clear before nominal planning equal priority --- .../astm/utm/clear_area_validation.md | 13 +++++ .../astm/utm/clear_area_validation.py | 52 ++++++++++++++++++ .../conflict_equal_priority_not_permitted.md | 6 +++ .../conflict_equal_priority_not_permitted.py | 16 +++++- .../scenarios/astm/utm/prep_planners.md | 27 ++-------- .../scenarios/astm/utm/prep_planners.py | 53 ++++--------------- .../uss_qualifier/suites/astm/utm/f3548_21.md | 2 +- .../suites/faa/uft/message_signing.md | 2 +- .../suites/uspace/flight_auth.md | 2 +- .../suites/uspace/required_services.md | 2 +- 10 files changed, 103 insertions(+), 72 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.py diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.md new file mode 100644 index 0000000000..06c6e9e011 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.md @@ -0,0 +1,13 @@ +# Clear area validation test step fragment + +uss_qualifier verifies with the DSS that there are no operational intents remaining in the area. + +## 🛑 DSS responses check + +If the DSS fails to reply to a query concerning operational intent references in a given area, or fails to allow the deletion of +an operational intent from its own creator, it is in violation of **[astm.f3548.v21.DSS0005,1](../../../requirements/astm/f3548/v21.md)** +or **[astm.f3548.v21.DSS0005,2](../../../requirements/astm/f3548/v21.md)**, and this check will fail. + +## 🛑 Area is clear of op intents check + +If operational intents exist in the 4D area(s) that should be clear, then the current state of the test environment is not suitable to conduct tests so this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.py new file mode 100644 index 0000000000..95d963219a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/clear_area_validation.py @@ -0,0 +1,52 @@ +from typing import List + +from monitoring.monitorlib.fetch import QueryError +from monitoring.monitorlib.geotemporal import Volume4D +from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from uas_standards.astm.f3548.v21.api import OperationalIntentReference + + +def validate_clear_area( + scenario: TestScenario, + dss: DSSInstance, + areas: List[Volume4D], + ignore_self: bool, +) -> List[OperationalIntentReference]: + found_intents = [] + for area in areas: + with scenario.check("DSS responses", [dss.participant_id]) as check: + try: + op_intents, query = dss.find_op_intent(area.to_f3548v21()) + scenario.record_query(query) + except QueryError as e: + scenario.record_queries(e.queries) + query = e.queries[0] + check.record_failed( + summary="Error querying DSS for operational intents", + details=f"See query; {e}", + query_timestamps=[query.request.timestamp], + ) + found_intents.extend(op_intents) + + with scenario.check("Area is clear of op intents") as check: + if ignore_self: + uss_qualifier_sub = dss.client.auth_adapter.get_sub() + op_intents = [ + oi for oi in op_intents if oi.manager != uss_qualifier_sub + ] + if op_intents: + summary = f"{len(op_intents)} operational intent{'s' if len(op_intents) > 1 else ''} found in test area" + details = ( + "The following operational intents were observed even though the area was expected to be clear:\n" + + "\n".join( + f"* {oi.id} managed by {oi.manager}" for oi in op_intents + ) + ) + check.record_failed( + summary=summary, + details=details, + query_timestamps=[query.request.timestamp], + ) + + return found_intents diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md index ae7cdcf506..f1b43b2721 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md @@ -98,6 +98,12 @@ CMSA role in order to transition to the `Nonconforming` state in order to create DSSInstanceResource that provides access to a DSS instance where flight creation/sharing can be verified. +## Prerequisites check test case + +### [Verify area is clear test step](../../clear_area_validation.md) + +While this scenario assumes that the area used is already clear of any pre-existing flights (using, for instance, PrepareFlightPlanners scenario) in order to avoid a large number of area-clearing operations, the scenario will not proceed correctly if the area was left in a dirty state following a previous scenario that was supposed to leave the area clear. This test step verifies that the area is clear. + ## Attempt to plan flight into conflict test case ![Test case summary illustration](assets/attempt_to_plan_flight_into_conflict.svg) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index e4d03763a6..b250c503ce 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -8,12 +8,15 @@ validate_flight_intent_templates, ExpectedFlightIntent, ) +from monitoring.uss_qualifier.scenarios.astm.utm.clear_area_validation import ( + validate_clear_area, +) from monitoring.uss_qualifier.suites.suite import ExecutionContext from uas_standards.astm.f3548.v21.api import ( OperationalIntentReference, ) from uas_standards.astm.f3548.v21.constants import Scope -from monitoring.monitorlib.geotemporal import Volume4DCollection +from monitoring.monitorlib.geotemporal import Volume4DCollection, Volume4D from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance @@ -192,6 +195,17 @@ def run(self, context: ExecutionContext): f"{self.control_uss.config.participant_id}", ) + self.begin_test_case("Prerequisites check") + self.begin_test_step("Verify area is clear") + validate_clear_area( + self, + self.dss, + [Volume4D.from_f3548v21(self._intents_extent)], + ignore_self=True, + ) + self.end_test_step() + self.end_test_case() + self.begin_test_case("Attempt to plan flight into conflict") _ = self._attempt_plan_flight_conflict() self.end_test_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md index f897b7c6ac..bf61d6d7d5 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md @@ -60,19 +60,9 @@ All USSs are requested to remove all flights from the area under test. **[interuss.automated_testing.flight_planning.ClearArea](../../../requirements/interuss/automated_testing/flight_planning.md)** -### Clear area validation test step +### [Clear area validation test step](clear_area_validation.md) -uss_qualifier verifies with the DSS that there are no operational intents remaining in the area - -#### 🛑 DSS responses check - -If the DSS fails to reply to a query concerning operational intent references in a given area, or fails to allow the deletion of -an operational intent from its own creator, it is in violation of **[astm.f3548.v21.DSS0005,1](../../../requirements/astm/f3548/v21.md)** -or **[astm.f3548.v21.DSS0005,2](../../../requirements/astm/f3548/v21.md)**, and this check will fail. - -#### 🛑 Area is clear of foreign op intents check - -If operational intents from foreign (non-uss_qualifier) users remain in the 4D area(s) following the preceding area clearing, then the current state of the test environment is not suitable to conduct tests so this check will fail. +This step examines whether any operational intents remain. If any foreign (other than uss_qualifier-owned) operational intents remain, then this step's checks will fail. If any uss_qualifier-owned operational intents remain, the checks for this step do not fail but instead we proceed to the next test case. If the area is clear, we skip the next test case. ## uss_qualifier preparation test case @@ -84,15 +74,6 @@ In addition to foreign flight planners, uss_qualifier may have left operational The operational intent references managed by uss_qualifier discovered in the previous test case are removed. -### Clear area validation test step - -uss_qualifier verifies with the DSS that there are no operational intents remaining in the area. - -#### 🛑 DSS responses check - -If the DSS fails to reply to a query concerning operational intent references in a given area, it is in violation of **[astm.f3548.v21.DSS0005,1](../../../requirements/astm/f3548/v21.md)** -or **[astm.f3548.v21.DSS0005,2](../../../requirements/astm/f3548/v21.md)**, and this check will fail. - -#### 🛑 Area is clear check +### [Clear area validation test step](clear_area_validation.md) -If any operational intents remain in the 4D area(s) following the preceding area clearing, then the current state of the test environment is not suitable to conduct tests so this check will fail. +After removing the operational intents of all flight planning participants previously, and just having attempted to remove uss_qualifier-owned operational intents, the area should now be fully clear. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py index 1a10fa92c6..431ccd5a04 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py @@ -7,6 +7,9 @@ FlightPlannersResource, FlightIntentsResource, ) +from monitoring.uss_qualifier.scenarios.astm.utm.clear_area_validation import ( + validate_clear_area, +) from monitoring.uss_qualifier.scenarios.astm.utm.dss.test_step_fragments import ( remove_op_intent, ) @@ -63,8 +66,11 @@ def run(self, context): self.end_test_step() self.begin_test_step("Clear area validation") - remaining_op_intents = self._validate_clear_area( - "Area is clear of foreign op intents", ignore_self=True + remaining_op_intents = validate_clear_area( + self, + self.dss, + self.areas, + ignore_self=True, ) self.end_test_step() @@ -78,54 +84,13 @@ def run(self, context): self.end_test_step() self.begin_test_step("Clear area validation") - self._validate_clear_area("Area is clear", ignore_self=False) + validate_clear_area(self, self.dss, self.areas, ignore_self=False) self.end_test_step() self.end_test_case() self.end_test_scenario() - def _validate_clear_area( - self, check_name: str, ignore_self: bool - ) -> List[OperationalIntentReference]: - found_intents = [] - for area in self.areas: - with self.check("DSS responses", [self.dss.participant_id]) as check: - try: - op_intents, query = self.dss.find_op_intent(area.to_f3548v21()) - self.record_query(query) - except QueryError as e: - self.record_queries(e.queries) - query = e.queries[0] - check.record_failed( - summary="Error querying DSS for operational intents", - details=f"See query; {e}", - query_timestamps=[query.request.timestamp], - ) - found_intents.extend(op_intents) - - with self.check(check_name) as check: - if ignore_self: - uss_qualifier_sub = self.dss.client.auth_adapter.get_sub() - op_intents = [ - oi for oi in op_intents if oi.manager != uss_qualifier_sub - ] - if op_intents: - summary = f"{len(op_intents)} operational intent{'s' if len(op_intents) > 1 else ''} found in cleared area" - details = ( - "The following operational intents were observed even after clearing the area:\n" - + "\n".join( - f"* {oi.id} managed by {oi.manager}" for oi in op_intents - ) - ) - check.record_failed( - summary=summary, - details=details, - query_timestamps=[query.request.timestamp], - ) - - return found_intents - def _remove_my_op_intents( self, my_op_intents: List[OperationalIntentReference] ) -> None: diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 8291245a05..4684a52e4c 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -38,7 +38,7 @@ astm
.f3548
.v21 DSS0005,1 Implemented - ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted + ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted DSS0005,2 diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 1463da5f8e..b1e1295d09 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -21,7 +21,7 @@ astm
.f3548
.v21 DSS0005,1 Implemented - ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted + ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted DSS0005,2 diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index d20362a78e..a89e04e32f 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -22,7 +22,7 @@ astm
.f3548
.v21 DSS0005,1 Implemented - ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted + ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted DSS0005,2 diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index faa2760205..b1d18e28f6 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -457,7 +457,7 @@ astm
.f3548
.v21 DSS0005,1 Implemented - ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted + ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
ASTM SCD DSS: Interfaces authentication
ASTM SCD DSS: Operational Intent Reference Key Validation
ASTM SCD DSS: Operational Intent Reference Synchronization
ASTM SCD DSS: Subscription Simple
ASTM SCD DSS: Subscription Synchronization
ASTM SCD DSS: Subscription Validation
ASTM SCD DSS: Subscription and entity interaction
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted DSS0005,2