From 1c9caad6b7ff0c4ffaf8662452d6ad3f95308e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Tue, 19 Sep 2023 15:55:16 +0200 Subject: [PATCH 1/2] [uss_qualifier/scenarios/utm] Fix #178 --- .../flight_intent_validation.py | 24 +++++---- .../conflict_equal_priority_not_permitted.py | 21 +++++--- .../conflict_higher_priority.py | 27 ++++++++-- .../modify_activated_flight_intent.md | 10 ++-- .../prioritization_test_steps.py | 52 ++++++++++--------- .../scenarios/flight_planning/test_steps.py | 30 +++++++---- 6 files changed, 101 insertions(+), 63 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index dc018aac97..546f46b56b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -1,10 +1,12 @@ from uas_standards.astm.f3548.v21.api import OperationalIntentState from uas_standards.astm.f3548.v21.constants import OiMaxPlanHorizonDays +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightResponseResult, +) from monitoring.monitorlib import scd from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( Capability, - InjectFlightResult, ) from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance @@ -211,8 +213,8 @@ def _attempt_invalid(self): self, "Attempt to plan flight intent too far ahead of time", "Incorrectly planned", - {InjectFlightResult.Rejected}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Rejected}, + {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.invalid_too_far_away.request, ) @@ -229,8 +231,8 @@ def _attempt_invalid_offnominal(self): self, "Attempt to plan flight with an off-nominal volume", "Incorrectly planned", - {InjectFlightResult.Rejected}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Rejected}, + {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.invalid_accepted_offnominal.request, ) @@ -244,8 +246,8 @@ def _attempt_invalid_offnominal(self): self, "Attempt to modify planned flight with an off-nominal volume", "Incorrectly modified", - {InjectFlightResult.Rejected}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Rejected}, + {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.invalid_accepted_offnominal.request, ) @@ -271,8 +273,8 @@ def _attempt_invalid_offnominal(self): self, "Attempt to modify activated flight with an off-nominal volume", "Incorrectly modified", - {InjectFlightResult.Rejected}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Rejected}, + {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.invalid_activated_offnominal.request, ) @@ -332,8 +334,8 @@ def _validate_precision_intersection(self): self, "Attempt to plan flight conflicting by a tiny overlap", "Incorrectly planned", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.valid_conflict_tiny_overlap.request, ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index c108218b86..07067b43b6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -1,11 +1,13 @@ from typing import Optional from uas_standards.astm.f3548.v21.api import OperationalIntentState +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightResponseResult, +) from monitoring.monitorlib import scd from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( Capability, - InjectFlightResult, ) from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance @@ -397,13 +399,13 @@ def _modify_activated_flight_preexisting_conflict( self, "Declare flight 2 non-conforming", "Successful transition to non-conforming state", - {InjectFlightResult.Planned, InjectFlightResult.Rejected}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Planned, InjectFlightResponseResult.Rejected}, + {InjectFlightResponseResult.Failed: "Failure"}, self.control_uss, self.flight_2_equal_prio_nonconforming_time_range_A.request, self.flight_2_id, ) - if resp_flight_2.result == InjectFlightResult.Rejected: + if resp_flight_2.result == InjectFlightResponseResult.Rejected: msg = f"{self.control_uss.config.participant_id} rejected transition to a Nonconforming state because it does not support CMSA role, execution of the scenario was stopped without failure" self.record_note("Control USS does not support CMSA role", msg) raise ScenarioCannotContinueError(msg) @@ -421,14 +423,17 @@ def _modify_activated_flight_preexisting_conflict( self, "Attempt to modify activated flight 1 in conflict with activated flight 2", "Successful modification or rejection", - {InjectFlightResult.ReadyToFly, InjectFlightResult.Rejected}, - {InjectFlightResult.Failed: "Failure"}, + { + InjectFlightResponseResult.ReadyToFly, + InjectFlightResponseResult.Rejected, + }, + {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.flight_1_activated_time_range_A_extended.request, self.flight_1_id, ) - if resp_flight_1.result == InjectFlightResult.ReadyToFly: + if resp_flight_1.result == InjectFlightResponseResult.ReadyToFly: validate_shared_operational_intent( self, self.tested_uss, @@ -437,7 +442,7 @@ def _modify_activated_flight_preexisting_conflict( self.flight_1_activated_time_range_A_extended.request, resp_flight_1.operational_intent_id, ) - elif resp_flight_1.result == InjectFlightResult.Rejected: + elif resp_flight_1.result == InjectFlightResponseResult.Rejected: validate_shared_operational_intent( self, self.tested_uss, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py index cba0d78edc..fb0b8e1267 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py @@ -1,6 +1,9 @@ from typing import Optional from uas_standards.astm.f3548.v21.api import OperationalIntentState +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightResponseResult, +) from monitoring.monitorlib import scd from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( @@ -371,21 +374,37 @@ def _modify_activated_flight_conflict_preexisting(self) -> str: self.flight_2_id, ) - resp_flight_1 = modify_activated_flight_intent( + resp_flight_1_modif = modify_activated_flight_intent( self, "Modify activated flight 1 in conflict with activated flight 2", self.tested_uss, self.flight_1_activated_time_range_A_extended.request, self.flight_1_id, + preexisting_conflict=True, ) + # The tested USS may respond NotSupported to the modification of the activated flight in conflict. + # If that's the case, it will not impact the rest of the test scenario. + if ( + resp_flight_1_modif.result == InjectFlightResponseResult.NotSupported + or resp_flight_1_modif.operational_intent_id is None + ): + flight_1_op_intent_id = resp_flight_1.operational_intent_id + else: + flight_1_op_intent_id = resp_flight_1_modif.operational_intent_id + + if resp_flight_1_modif.result == InjectFlightResponseResult.NotSupported: + flight_1_op_intent = self.flight_1_activated_time_range_A + else: + flight_1_op_intent = self.flight_1_activated_time_range_A_extended + validate_shared_operational_intent( self, self.tested_uss, self.dss, "Validate flight 1 sharing", - self.flight_1_activated_time_range_A_extended.request, - resp_flight_1.operational_intent_id, + flight_1_op_intent.request, + flight_1_op_intent_id, ) validate_shared_operational_intent( self, @@ -396,7 +415,7 @@ def _modify_activated_flight_conflict_preexisting(self) -> str: resp_flight_2.operational_intent_id, ) - return resp_flight_1.operational_intent_id + return flight_1_op_intent_id def _attempt_modify_activated_flight_conflict(self, flight_1_op_intent_id: str): resp_flight_2 = modify_activated_flight_intent( diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md index f3d7773c15..3a21d1a21d 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md @@ -5,10 +5,12 @@ successfully modified by a flight planner. See `modify_activated_flight_intent` ## Successful modification check -All flight intent data provided is correct and valid and free of conflict in space and time, therefore it should have -been modified by the USS per **interuss.automated_testing.flight_planning.ExpectedBehavior**. -If the USS fails to modify the flight, wrongly indicates a conflict, or wrongly indicates the activated state of the -flight, this check will fail. +All flight intent data provided is correct and valid. The (already activated) provided flight intent may be in conflict with +another activated flight, but only if this conflict already existed before the modification was initiated. +Therefore, the USS should have either successfully modified the flight per **interuss.automated_testing.flight_planning.ExpectedBehavior**, +or indicated that the operation is not supported. +If the USS fails to modify the flight (or to indicate that the modification is not supported), wrongly indicates a +conflict, or wrongly indicates the activated state of the flight, this check will fail. ## Failure check diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py index 5b6d9285a2..548fce9a59 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py @@ -1,10 +1,12 @@ from typing import Optional, Tuple from uas_standards.astm.f3548.v21.api import OperationalIntentState +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightResponseResult, +) from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( InjectFlightRequest, - InjectFlightResult, InjectFlightResponse, ) from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( @@ -38,8 +40,8 @@ def plan_priority_conflict_flight_intent( scenario, test_step, "Incorrectly planned", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, )[0] @@ -67,8 +69,8 @@ def modify_planned_priority_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -97,8 +99,8 @@ def activate_priority_conflict_flight_intent( scenario, test_step, "Incorrectly activated", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -127,8 +129,8 @@ def modify_activated_priority_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -156,8 +158,8 @@ def plan_conflict_flight_intent( scenario, test_step, "Incorrectly planned", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, )[0] @@ -185,8 +187,8 @@ def modify_planned_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -215,8 +217,8 @@ def activate_conflict_flight_intent( scenario, test_step, "Incorrectly activated", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -245,8 +247,8 @@ def modify_activated_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResult.ConflictWithFlight}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ConflictWithFlight}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -276,8 +278,8 @@ def plan_permitted_conflict_flight_intent( scenario, test_step, "Successful planning", - {InjectFlightResult.Planned}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Planned}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, ) @@ -305,8 +307,8 @@ def modify_planned_permitted_conflict_flight_intent( scenario, test_step, "Successful modification", - {InjectFlightResult.Planned}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Planned}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -335,8 +337,8 @@ def activate_permitted_conflict_flight_intent( scenario, test_step, "Successful activation", - {InjectFlightResult.ReadyToFly}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ReadyToFly}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -365,8 +367,8 @@ def modify_activated_permitted_conflict_flight_intent( scenario, test_step, "Successful modification", - {InjectFlightResult.ReadyToFly}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ReadyToFly}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 9bf85ad654..6c7f2cc58b 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -3,13 +3,15 @@ from typing import List, Union, Optional, Tuple, Iterable, Set, Dict from uas_standards.astm.f3548.v21.api import OperationalIntentState +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightResponseResult, +) from monitoring.monitorlib.fetch import QueryError from monitoring.monitorlib.scd import bounding_vol4 from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( InjectFlightRequest, Capability, - InjectFlightResult, InjectFlightResponse, DeleteFlightResult, DeleteFlightResponse, @@ -254,8 +256,8 @@ def plan_flight_intent( scenario, test_step, "Successful planning", - {InjectFlightResult.Planned}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Planned}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, ) @@ -283,8 +285,8 @@ def activate_flight_intent( scenario, test_step, "Successful activation", - {InjectFlightResult.ReadyToFly}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.ReadyToFly}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -313,8 +315,8 @@ def modify_planned_flight_intent( scenario, test_step, "Successful modification", - {InjectFlightResult.Planned}, - {InjectFlightResult.Failed: "Failure"}, + {InjectFlightResponseResult.Planned}, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -327,8 +329,11 @@ def modify_activated_flight_intent( flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, + preexisting_conflict: bool = False, ) -> InjectFlightResponse: """Modify an activated flight intent that should result in success. + If the activated flight intent to modify has a pre-existing conflict, the USS is allowed to return `NotSupported`. + Use `preexisting_conflict=True` in this case. This function implements the test step described in modify_activated_flight_intent.md. @@ -339,12 +344,15 @@ def modify_activated_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) + expected_results = {InjectFlightResponseResult.ReadyToFly} + if preexisting_conflict: + expected_results.add(InjectFlightResponseResult.NotSupported) return submit_flight_intent( scenario, test_step, "Successful modification", - {InjectFlightResult.ReadyToFly}, - {InjectFlightResult.Failed: "Failure"}, + expected_results, + {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, flight_id, @@ -355,8 +363,8 @@ def submit_flight_intent( scenario: TestScenarioType, test_step: str, success_check: str, - expected_results: Set[InjectFlightResult], - failed_checks: Dict[InjectFlightResult, str], + expected_results: Set[InjectFlightResponseResult], + failed_checks: Dict[InjectFlightResponseResult, str], flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, From 71c354441ce72cbb6fa9789f7cb1d3ffdd60603b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Fri, 6 Oct 2023 11:09:57 +0200 Subject: [PATCH 2/2] add req for op intent coverage; update according to feedback; optional severity --- .../automated_testing/flight_planning.md | 5 ++ .../flight_intent_validation.py | 6 +-- .../conflict_higher_priority.md | 27 ++++++++-- .../conflict_higher_priority.py | 8 ++- .../scenarios/astm/utm/test_steps.py | 35 ++++++++++--- .../utm/validate_shared_operational_intent.md | 6 +++ .../modify_activated_flight_intent.md | 40 ++++++++++++--- .../scenarios/flight_planning/test_steps.py | 51 ++++++++++++++----- .../uss_qualifier/suites/astm/utm/f3548_21.md | 7 ++- .../suites/faa/uft/message_signing.md | 7 ++- .../suites/uspace/flight_auth.md | 7 ++- .../suites/uspace/required_services.md | 7 ++- 12 files changed, 159 insertions(+), 47 deletions(-) diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md index 7e4f6ae599..63c7169b31 100644 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md @@ -12,4 +12,9 @@ TODO: Describe requirements ### ExpectedBehavior +### FlightCoveredByOperationalIntent +For InterUSS to effectively test the requirements of ASTM F3548-21, a USS under test must act as if there is a +regulatory requirement requiring all flights it manages to provide operational intents according to ASTM F3548-21 at all +times for all flights it manages. + ### DeleteFlightSuccess diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index 59eec2e0ad..226cb91a99 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -2,9 +2,6 @@ from monitoring.uss_qualifier.common_data_definitions import Severity from uas_standards.astm.f3548.v21.api import OperationalIntentState from uas_standards.astm.f3548.v21.constants import OiMaxPlanHorizonDays -from uas_standards.interuss.automated_testing.scd.v1.api import ( - InjectFlightResponseResult, -) from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance @@ -32,6 +29,9 @@ submit_flight_intent, delete_flight_intent, ) +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightResponseResult, +) class FlightIntentValidation(TestScenario): diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md index 4c446e4049..15e012b15f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md @@ -174,13 +174,30 @@ The test driver activates flight 2, which should be done successfully given that ### [Validate flight 2 sharing test step](../../validate_shared_operational_intent.md) ### [Modify activated flight 1 in conflict with activated flight 2 test step](../../../../flight_planning/modify_activated_flight_intent.md) -Before execution of this step, flights 1 and 2 are activated and in conflict. -The test driver modifies flight 1 in a way that still conflicts with flight 2. -Even though flight 2 is the highest-priority flight, because the conflict existed before the modification was initiated, -the modification is accepted per **[astm.f3548.v21.SCD0030](../../../../../requirements/astm/f3548/v21.md)**. +Before execution of this step, flights 1 and 2 are activated and in conflict. Flight 2 is the highest-priority flight. +The test driver attempts to modify flight 1 in a way that still conflicts with flight 2. + +The successful outcomes of the modification attempts: +1. Even though flight 2 is the highest-priority flight, because the conflict existed before the modification was + initiated, an accepted modification is considered a success per **[astm.f3548.v21.SCD0030](../../../../../requirements/astm/f3548/v21.md)**. +2. Due to the conflict, the USS may decide to be more conservative and to not support the modification. This is + considered a success as there is no positive requirement for the USS to accept the modification. + +A rejected modification will indicate a low severity failure. Indeed, in some situations a rejection may not be strictly +speaking a failure to meet a requirement. This could be the case for example if the USS does not support directly update +of intents and instead delete the previous one and create a new one. Since we cannot distinguish between an actual +failure to meet the requirement and a reasonable behavior due to implementation limitations, we indicate a low severity +failure which won't actually fail the test. + +In any case, whatever is the outcome of this step, there should not be any impact on the rest of the execution of the +scenario. An intent should exist (this is checked in the next step) and it should be either the previous or the modified +intent, both of which make no difference in the next steps. ### [Validate flight 1 sharing test step](../../validate_shared_operational_intent.md) -The first flight should have been modified. +If the modification was accepted, flight 1 should have been modified. +If the modification was not supported, flight 1 should not have been modified. +If the modification was rejected, flight 1 should not have been modified and should still exist. If it does not exist, +it means that there is an active flight without an operational intent, which is a failure to meet **[interuss.automated_testing.flight_planning.FlightCoveredByOperationalIntent](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. ## Attempt to modify activated flight in conflict test case diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py index 733d4a993a..165694c73a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py @@ -451,15 +451,13 @@ def _modify_activated_flight_conflict_preexisting( preexisting_conflict=True, ) - # The tested USS may respond NotSupported to the modification of the activated flight in conflict. - # If that's the case, it will not impact the rest of the test scenario. - if resp.result == InjectFlightResponseResult.NotSupported: + if resp.result == InjectFlightResponseResult.ReadyToFly: flight_1_oi_ref = validator.expect_shared( - self.flight_1_activated_time_range_A.request + self.flight_1_activated_time_range_A_extended.request ) else: flight_1_oi_ref = validator.expect_shared( - self.flight_1_activated_time_range_A_extended.request + self.flight_1_activated_time_range_A.request ) return flight_1_oi_ref, flight_2_oi_ref diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index ca7f590eed..46bf3e9c60 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -168,17 +168,36 @@ def expect_shared( oi_ref = self._new_oi_ref elif self._new_oi_ref is None: - # we expect the original op intent to have been either modified or left untouched, thus must be among the returned op intents - # exception made if skip_if_not_found=True and op intent was deleted: step is skipped + # We expect the original op intent to have been either modified or left untouched, thus must be among + # the returned op intents. If additionally the op intent corresponds to an active flight, we fail a + # different appropriate check. Exception made if skip_if_not_found=True and op intent was deleted: step + # is skipped. modified_oi_ref = self._find_after_oi(self._orig_oi_ref.id) if modified_oi_ref is None: if not skip_if_not_found: - check.record_failed( - summary="Operational intent reference not found in DSS", - severity=Severity.High, - details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", - query_timestamps=[self._after_query.request.timestamp], - ) + if ( + flight_intent.operational_intent.state + == OperationalIntentState.Activated + ): + with self._scenario.check( + "Operational intent for active flight not deleted", + [self._flight_planner.participant_id], + ) as active_flight_check: + active_flight_check.record_failed( + summary="Operational intent reference for active flight not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[ + self._after_query.request.timestamp + ], + ) + else: + check.record_failed( + summary="Operational intent reference not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[self._after_query.request.timestamp], + ) else: self._scenario.record_note( self._flight_planner.participant_id, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md index 35430975b7..b6fac6fae6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md @@ -10,6 +10,12 @@ This step verifies that a created flight is shared properly per ASTM F3548-21 by If a reference to the operational intent for the flight is not found in the DSS, this check will fail per **astm.f3548.v21.USS0005** and **astm.f3548.v21.OPIN0025**. +## Operational intent for active flight not deleted check + +If an activated operational intent is expected to exist after it has been modified or activated and that it is not found +in the DSS, this means that there is an active flight without a corresponding operational intent, then this check will +fail per **[interuss.automated_testing.flight_planning.FlightCoveredByOperationalIntent](../../../requirements/interuss/automated_testing/flight_planning.md)**. + ## Operational intent details retrievable check If the operational intent details for the flight cannot be retrieved from the USS, this check will fail per **astm.f3548.v21.USS0105** and **astm.f3548.v21.OPIN0025**. diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md index 3a21d1a21d..9b7db65ab2 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_flight_intent.md @@ -1,16 +1,40 @@ # Modify activated flight test step -This page describes the content of a common test case where a valid user flight intent in activated state should be -successfully modified by a flight planner. See `modify_activated_flight_intent` in [test_steps.py](test_steps.py). +This page describes the content of a common test case where a valid user flight intent in activated state is tentatively +modified by a flight planned. Multiple outcomes may be valid. +See `modify_activated_flight_intent` in [test_steps.py](test_steps.py). ## Successful modification check -All flight intent data provided is correct and valid. The (already activated) provided flight intent may be in conflict with -another activated flight, but only if this conflict already existed before the modification was initiated. -Therefore, the USS should have either successfully modified the flight per **interuss.automated_testing.flight_planning.ExpectedBehavior**, -or indicated that the operation is not supported. -If the USS fails to modify the flight (or to indicate that the modification is not supported), wrongly indicates a -conflict, or wrongly indicates the activated state of the flight, this check will fail. +All flight intent data provided is correct and valid. The (already activated) provided flight intent may be in conflict +with another activated flight, but only if this conflict already existed before the modification was initiated. + +If the provided flight intent is not in conflict with another intent the USS should have successfully modified the +flight per **[astm.f3548.v21.SCD0030](../../requirements/astm/f3548/v21.md)**. +If the USS fails to modify the flight, wrongly indicates a conflict, or wrongly indicates the activated state of the +flight, this check will fail. + +If the provided flight intent is in conflict with another intent and that a pre-existing conflict was present, the USS +may have decided to be more conservative and to not support modification. +In such case, the USS may indicate that the operation is not supported instead of modifying the flight per **[astm.f3548.v21.SCD0030](../../requirements/astm/f3548/v21.md)**. +If the USS fails to modify the flight, or fails to indicate that the modification is not supported, or wrongly indicates +the activated state of the flight, this check will fail. + +Do take note that if the USS rejects the modification when a pre-existing conflict was present, this check will not fail, +but the following *Rejected modification check* will. Refer to this check for more information. + +## Rejected modification check + +If the provided flight intent is in conflict with another intent and that a pre-existing conflict was present, the USS +may have rejected the modification instead of modifying it or indicating that the modification is not supported. This +could be the case for example if the USS does not support directly update of intents and instead delete the previous one +and create a new one. This may or may not be strictly speaking a failure to meet a requirement, but we cannot +distinguish between an actual failure to meet the requirement and a reasonable behavior due to implementation +limitations. + +As such, if the pre-existing conflict was present, and that the USS rejected the modification, this check will fail with +a low severity per **[astm.f3548.v21.SCD0030](../../requirements/astm/f3548/v21.md)**. This won't actually fail the test +but will serve as a warning. ## Failure check diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 40804df96a..502a7f2eff 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -1,11 +1,8 @@ import inspect -from typing import List, Optional, Tuple, Iterable, Set, Dict +from typing import List, Optional, Tuple, Iterable, Set, Dict, Union from monitoring.monitorlib.geotemporal import Volume4DCollection from uas_standards.astm.f3548.v21.api import OperationalIntentState -from uas_standards.interuss.automated_testing.scd.v1.api import ( - InjectFlightResponseResult, -) from monitoring.monitorlib.fetch import QueryError from uas_standards.interuss.automated_testing.scd.v1.api import ( @@ -187,9 +184,8 @@ def modify_activated_flight_intent( flight_id: str, preexisting_conflict: bool = False, ) -> InjectFlightResponse: - """Modify an activated flight intent that should result in success. - If the activated flight intent to modify has a pre-existing conflict, the USS is allowed to return `NotSupported`. - Use `preexisting_conflict=True` in this case. + """Attempt to modify an activated flight intent. + If present, a pre-existing conflict must be indicated with `preexisting_conflict=True`. This function implements the test step described in modify_activated_flight_intent.md. @@ -200,15 +196,35 @@ def modify_activated_flight_intent( flight_intent, OperationalIntentState.Activated, scenario, test_step ) - expected_results = {InjectFlightResponseResult.ReadyToFly} if preexisting_conflict: - expected_results.add(InjectFlightResponseResult.NotSupported) + expected_results = { + InjectFlightResponseResult.ReadyToFly, + InjectFlightResponseResult.NotSupported, + # the following two results are considered expected in order to fail another check as low severity + InjectFlightResponseResult.Rejected, + InjectFlightResponseResult.ConflictWithFlight, + } + failed_checks = { + InjectFlightResponseResult.Failed: "Failure", + InjectFlightResponseResult.Rejected: ( + "Rejected modification", + Severity.Low, + ), + InjectFlightResponseResult.ConflictWithFlight: ( + "Rejected modification", + Severity.Low, + ), + } + else: + expected_results = {InjectFlightResponseResult.ReadyToFly} + failed_checks = {InjectFlightResponseResult.Failed: "Failure"} + return submit_flight_intent( scenario, test_step, "Successful modification", expected_results, - {InjectFlightResponseResult.Failed: "Failure"}, + failed_checks, flight_planner, flight_intent, flight_id, @@ -220,13 +236,14 @@ def submit_flight_intent( test_step: str, success_check: str, expected_results: Set[InjectFlightResponseResult], - failed_checks: Dict[InjectFlightResponseResult, str], + failed_checks: Dict[InjectFlightResponseResult, Union[str, Tuple[str, Severity]]], flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, ) -> Tuple[InjectFlightResponse, Optional[str]]: """Submit a flight intent with an expected result. - A check fail is considered of high severity and as such will raise an ScenarioCannotContinueError. + A check fail is considered by default of high severity and as such will raise an ScenarioCannotContinueError. + The severity of each failed check may be overridden if needed. This function does not directly implement a test step. @@ -253,13 +270,19 @@ def submit_flight_intent( notes_suffix = f': "{resp.notes}"' if "notes" in resp and resp.notes else "" for unexpected_result, failed_test_check in failed_checks.items(): + if isinstance(failed_test_check, str): + check_name = failed_test_check + check_severity = Severity.High + else: + check_name, check_severity = failed_test_check + with scenario.check( - failed_test_check, [flight_planner.participant_id] + check_name, [flight_planner.participant_id] ) as specific_failed_check: if resp.result == unexpected_result: specific_failed_check.record_failed( summary=f"Flight unexpectedly {resp.result}", - severity=Severity.High, + severity=check_severity, details=f'{flight_planner.participant_id} indicated {resp.result} rather than the expected {" or ".join(expected_results)}{notes_suffix}', query_timestamps=[query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index c3f4c1c418..9b83332c44 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -112,7 +112,7 @@ Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents - interuss
.automated_testing
.flight_planning
+ interuss
.automated_testing
.flight_planning
ClearArea Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -127,4 +127,9 @@ Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + + FlightCoveredByOperationalIntent + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 7efaf4f0f9..050dde31a4 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -109,7 +109,7 @@ Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents - interuss
.automated_testing
.flight_planning
+ interuss
.automated_testing
.flight_planning
ClearArea Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -124,4 +124,9 @@ Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + + FlightCoveredByOperationalIntent + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 93884674fc..f7dd88ba0a 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -109,7 +109,7 @@ Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents - interuss
.automated_testing
.flight_planning
+ interuss
.automated_testing
.flight_planning
ClearArea Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -124,4 +124,9 @@ Implemented Flight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + + FlightCoveredByOperationalIntent + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index ac5f5092ea..a690bb7ab8 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -479,7 +479,7 @@ Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents - interuss
.automated_testing
.flight_planning
+ interuss
.automated_testing
.flight_planning
ClearArea Implemented Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents @@ -494,6 +494,11 @@ Implemented Flight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + + FlightCoveredByOperationalIntent + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents + interuss
.automated_testing
.rid
.injection
DeleteTestSuccess