diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 61af0ba764..7ed65a134d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -29,18 +29,22 @@ DSSInstanceResource that provides access to a DSS instance where flight creation ## Successfully plan flight near an existing flight test case -### [mock_uss plans flight 2 test step](../../../flight_planning/plan_flight_intent.md) +### mock_uss plans flight 2 test step + +#### [Plan](../../../flight_planning/plan_flight_intent.md) + Flight 2 should be successfully planned by the control USS. -### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) -Validate that flight 2 is planned +#### [Validate](../validate_shared_operational_intent.md) + +### tested_uss plans flight 1 test step + +#### [Plan](../../../flight_planning/plan_flight_intent.md) -### [tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 which is of equal priority and came first. -### [Validate flight 1 sharing test step](../validate_shared_operational_intent.md) -Validate flight 1 is planned. +#### [Validate](../validate_shared_operational_intent.md) ### Check for notification to tested_uss due to subscription in flight 2 area test step In the following test step, we want to assert that tested_uss must have retrieved operational intent details from @@ -63,18 +67,27 @@ Teardown Teardown ## Flight planning prevented due to invalid data sharing test case -### [mock_uss plans flight 2, sharing invalid operational intent data test step](../../../flight_planning/plan_flight_intent.md) + +### mock_uss plans flight 2, sharing invalid operational intent data test step + +#### [Plan](../../../flight_planning/plan_flight_intent.md) + Flight 2 should be successfully planned by the mock_uss. + +#### [Validate](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) + The mock_uss is instructed to share invalid data with other USS, for negative test. -### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) -Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. +### tested_uss attempts to plan flight 1, expect failure test step + +#### [Plan](test_steps/plan_flight_intent_expect_failed.md) -### [tested_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. +The planning attempt should fail because tested_uss will be unable to obtain valid operational intent details for flight 2. + +#### [Validate](../validate_not_shared_operational_intent.md) -### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) Validate flight 1 is not shared with DSS, as plan failed. ### Check for notification to tested_uss due to subscription in flight 2 area test step diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index 05294e0cd2..b8974d5731 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -167,45 +167,43 @@ def _plan_successfully_test_case(self, times: Dict[TimeDuringTest, Time]): times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_2 = self.flight_2.resolve(times) + self.begin_test_step("mock_uss plans flight 2") with OpIntentValidator( self, self.mock_uss_client, self.dss, - "Validate flight 2 sharing", self._intents_extent, ) as validator: - self.begin_test_step("mock_uss plans flight 2") flight_2_planning_time = Time(arrow.utcnow().datetime) _, self.flight_2_id = plan_flight( self, self.mock_uss_client, flight_2, ) - self.end_test_step() flight_2_oi_ref = validator.expect_shared(flight_2) + self.end_test_step() times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_1 = self.flight_1.resolve(times) + self.begin_test_step("tested_uss plans flight 1") with OpIntentValidator( self, self.tested_uss_client, self.dss, - "Validate flight 1 sharing", self._intents_extent, ) as validator: - self.begin_test_step("tested_uss plans flight 1") flight_1_planning_time = Time(arrow.utcnow().datetime) plan_res, self.flight_1_id = plan_flight( self, self.tested_uss_client, flight_1, ) - self.end_test_step() validator.expect_shared( flight_1, ) + self.end_test_step() self.begin_test_step( "Check for notification to tested_uss due to subscription in flight 2 area" @@ -281,16 +279,15 @@ def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]): ) additional_fields = {"behavior": behavior} + self.begin_test_step( + "mock_uss plans flight 2, sharing invalid operational intent data" + ) with OpIntentValidator( self, self.mock_uss_client, self.dss, - "Validate flight 2 shared operational intent with invalid data", self._intents_extent, ) as validator: - self.begin_test_step( - "mock_uss plans flight 2, sharing invalid operational intent data" - ) flight_2_planning_time = Time(arrow.utcnow().datetime) _, self.flight_2_id = plan_flight( self, @@ -298,31 +295,30 @@ def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]): flight_info, additional_fields, ) - self.end_test_step() flight_2_oi_ref = validator.expect_shared_with_invalid_data( flight_info, validation_failure_type=OpIntentValidationFailureType.DataFormat, invalid_fields=[modify_field1, modify_field2], ) + self.end_test_step() times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_1 = self.flight_1.resolve(times) + self.begin_test_step("tested_uss attempts to plan flight 1, expect failure") with OpIntentValidator( self, self.tested_uss_client, self.dss, - "Validate flight 1 not shared by tested_uss", self._intents_extent, ) as validator: - self.begin_test_step("tested_uss attempts to plan flight 1, expect failure") flight_1_planning_time = Time(arrow.utcnow().datetime) _, self.flight_1_id = plan_flight_intent_expect_failed( self, self.tested_uss_client, flight_1, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() self.begin_test_step( "Check for notification to tested_uss due to subscription in flight 2 area" diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md index f606904927..f0eb4a2666 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md @@ -44,19 +44,23 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate flight intent too far ahead of time not planned test step](../validate_not_shared_operational_intent.md) +#### [Validate flight intent too far ahead of time not planned](../validate_not_shared_operational_intent.md) ## Validate transition to Ended state after cancellation test case -### [Plan flight intent test step](../../../flight_planning/plan_flight_intent.md) +### Plan flight intent test step + +#### [Plan](../../../flight_planning/plan_flight_intent.md) The valid flight intent should be successfully planned by the flight planner. -### [Validate flight intent shared correctly test step](../validate_shared_operational_intent.md) +#### [Validate flight intent shared correctly](../validate_shared_operational_intent.md) Validate that the flight intent was shared correctly and is discoverable. -### [Cancel flight intent test step](../../../flight_planning/delete_flight_intent.md) +### Remove flight intent test step + +#### [Cancel flight intent](../../../flight_planning/delete_flight_intent.md) The flight intent should be successfully transition to Ended state by the flight planner. -### Validate flight intent is non-discoverable test step +#### [Validate flight intent is non-discoverable](../validate_removed_operational_intent.md) #### DSS responses check @@ -84,7 +88,7 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate conflicting flight not planned test step](../validate_not_shared_operational_intent.md) +#### [Validate conflicting flight not planned](../validate_not_shared_operational_intent.md) ## Cleanup ### Successful flight deletion check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index 8434ff2aec..c0819077e5 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -165,14 +165,13 @@ def run(self, context: ExecutionContext): self.end_test_scenario() def _attempt_invalid(self): + self.begin_test_step("Attempt to plan flight intent too far ahead of time") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate flight intent too far ahead of time not planned", self._intents_extent, ) as validator: - self.begin_test_step("Attempt to plan flight intent too far ahead of time") submit_flight_intent( self, "Incorrectly planned", @@ -181,38 +180,36 @@ def _attempt_invalid(self): self.tested_uss, self.invalid_too_far_away.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() def _validate_ended_cancellation(self): + self.begin_test_step("Plan flight intent") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate flight intent is non-discoverable", self._intents_extent, - ) as cancelled_validator: - with OpIntentValidator( + ) as planned_validator: + _, flight_id, _ = plan_flight_intent( self, self.tested_uss, - self.dss, - "Validate flight intent shared correctly", - self._intents_extent, - ) as planned_validator: - self.begin_test_step("Plan flight intent") - _, flight_id, _ = plan_flight_intent( - self, - self.tested_uss, - self.valid_flight.request, - ) - self.end_test_step() - planned_validator.expect_shared(self.valid_flight.request) - - _ = delete_flight_intent( - self, "Cancel flight intent", self.tested_uss, flight_id + self.valid_flight.request, ) - cancelled_validator.expect_not_shared() + oi_ref = planned_validator.expect_shared(self.valid_flight.request) + self.end_test_step() + + self.begin_test_step("Remove flight intent") + with OpIntentValidator( + self, + self.tested_uss, + self.dss, + self._intents_extent, + ) as cancelled_validator: + _ = delete_flight_intent(self, self.tested_uss, flight_id) + cancelled_validator.expect_removed(oi_ref.id) + self.end_test_step() def _validate_precision_intersection(self): self.begin_test_step("Plan control flight intent") @@ -223,14 +220,13 @@ def _validate_precision_intersection(self): ) self.end_test_step() + self.begin_test_step("Attempt to plan flight conflicting by a tiny overlap") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate conflicting flight not planned", self._intents_extent, ) as validator: - self.begin_test_step("Attempt to plan flight conflicting by a tiny overlap") submit_flight_intent( self, "Incorrectly planned", @@ -242,9 +238,9 @@ def _validate_precision_intersection(self): self.tested_uss, self.valid_conflict_tiny_overlap.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() def cleanup(self): self.begin_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md index b4eae2886c..ae7cdcf506 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md @@ -106,67 +106,81 @@ DSSInstanceResource that provides access to a DSS instance where flight creation #### [Plan Flight 2](../../../../flight_planning/plan_flight_intent.md) Flight 2 should be successfully planned by the control USS. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) -### [Activate Flight 2 test step](../../../../flight_planning/activate_flight_intent.md) +### Activate Flight 2 test step + +#### [Activate Flight 2](../../../../flight_planning/activate_flight_intent.md) Flight 2 should be successfully activated by the control USS. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) + +### Attempt to plan Flight 1 test step -### [Attempt to plan Flight 1 test step](../../../../flight_planning/plan_conflict_flight_intent.md) +#### [Attempt to plan Flight 1](../../../../flight_planning/plan_conflict_flight_intent.md) The test driver attempts to plan the Flight 1 via the tested USS. However, it conflicts with Flight 2 which is of equal priority but came first. As such it should be rejected per **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1 not shared test step](../../validate_not_shared_operational_intent.md) +#### [Validate Flight 1 not shared](../../validate_not_shared_operational_intent.md) Flight 1 should not have been shared with the interoperability ecosystem since it was rejected. ## Attempt to activate flight into conflict test case ![Test case summary illustration](assets/attempt_to_activate_flight_into_conflict.svg) -### [Attempt to directly activate conflicting Flight 1 test step](../../../../flight_planning/activate_conflict_flight_intent.md) +### Attempt to directly activate conflicting Flight 1 test step + +#### [Attempt to activate Flight 1](../../../../flight_planning/activate_conflict_flight_intent.md) The test driver attempts to activate directly Flight 1, i.e. without the flight being planned before. However, this conflicts with activated Flight 2, which is of equal priority. As such it should be rejected per **[astm.f3548.v21.SCD0045](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1 not shared test step](../../validate_not_shared_operational_intent.md) +#### [Validate Flight 1 not shared](../../validate_not_shared_operational_intent.md) Flight 1 should not have been shared with the interoperability ecosystem since it was rejected. ## Attempt to modify planned flight into conflict test case ![Test case summary illustration](assets/attempt_to_modify_planned_flight_into_conflict.svg) -### [Plan Flight 1c test step](../../../../flight_planning/plan_flight_intent.md) +### Plan Flight 1c test step + +#### [Plan Flight 1c](../../../../flight_planning/plan_flight_intent.md) The smaller Flight 1c form (which doesn't conflict with Flight 2) should be successfully planned by the tested USS. -### [Validate Flight 1c sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1c sharing](../../validate_shared_operational_intent.md) + +### Attempt to modify planned Flight 1c into conflict test step -### [Attempt to modify planned Flight 1c into conflict test step](../../../../flight_planning/modify_planned_conflict_flight_intent.md) +#### [Attempt to modify Flight 1c](../../../../flight_planning/modify_planned_conflict_flight_intent.md) The test driver attempts to enlarge Flight 1c so that it conflicts with Flight 2. However, Flight 2 is of equal priority but was planned first. As such the change to Flight 1c should be rejected per **[astm.f3548.v21.SCD0040](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1c not modified test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1c not modified](../../validate_shared_operational_intent.md) Because the modification attempt was invalid, either Flight 1c should not have been modified (because the USS kept the original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). ## Attempt to modify activated flight into conflict test case ![Test case summary illustration](assets/attempt_to_modify_activated_flight_into_conflict.svg) -### [Activate Flight 1c test step](../../../../flight_planning/activate_flight_intent.md) +### Activate Flight 1c test step + +#### [Activate Flight 1c](../../../../flight_planning/activate_flight_intent.md) The test driver activates the smaller Flight 1c form, which should be done successfully. Note that Flight 1c could be either planned or non-existent before this step. In the latter case, the "user" will directly activate the flight without planning it beforehand. -### [Validate Flight 1c sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1c sharing](../../validate_shared_operational_intent.md) + +### Attempt to modify activated Flight 1c into conflict test step -### [Attempt to modify activated Flight 1c into conflict test step](../../../../flight_planning/modify_activated_conflict_flight_intent.md) +#### [Attempt to modify Flight 1c](../../../../flight_planning/modify_activated_conflict_flight_intent.md) The test driver attempts to enlarge Flight 1c so that it conflicts with Flight 2. Both flights are activated at the point where that change is requested. However, because the conflict did not exist when the modification was initiated, it should be rejected per **[astm.f3548.v21.SCD0050](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1c not modified test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1c not modified](../../validate_shared_operational_intent.md) Because the modification attempt was invalid, either Flight 1c should not have been modified (because the USS kept the original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). @@ -176,16 +190,20 @@ To prepare for the next test case, Flight 2 must be removed from the system. ## Modify activated flight with pre-existing conflict test case ![Test case summary illustration](assets/modify_activated_flight_with_preexisting_conflict.svg) -### [Activate Flight 1 test step](../../../../flight_planning/activate_flight_intent.md) +### Activate Flight 1 test step + +#### [Activate Flight 1](../../../../flight_planning/activate_flight_intent.md) The test driver activates the normal form of Flight 1, which should be done successfully since Flight 2 was removed at the end of the previous step. Note that Flight 1 could be either already activated in the smaller Flight 1c form or non-existent before this step. In the former case, Flight 1c is enlarged to its normal size. In the latter case, the flight will be directly activated without being planned beforehand. -### [Validate Flight 1 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 sharing](../../validate_shared_operational_intent.md) + +### Plan Flight 2m test step -### [Plan Flight 2m test step](../../../../flight_planning/plan_flight_intent.md) +#### [Plan Flight 2m](../../../../flight_planning/plan_flight_intent.md) The smaller Flight 2 form should be successfully planned by the control USS because it does not conflict with Flight 1. -### [Validate Flight 2m sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2m sharing](../../validate_shared_operational_intent.md) ### Declare Flight 2 non-conforming test step The test driver instructs the control USS to declare Flight 2 as non-conforming. This makes non-conforming Flight 2 conflict with activated Flight 1 -- this same-priority conflict would not be allowed if Flight 2 were in a nominal state. @@ -205,7 +223,7 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) ### Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2 test step Before execution of this step, Flight 1 is activated (onto time range A) and Flight 2 is non-conforming (onto time range @@ -226,7 +244,7 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate Flight 1 test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 sharing](../../validate_shared_operational_intent.md) This step validates that the response of the USS is consistent with the flight shared, i.e. either it was properly modified, or the USS considered the attempt invalid. In the latter case, because the modification attempt was invalid, either Flight 1 should not have been modified (because the USS kept the original accepted request), or it should have diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index 1a777d566f..e4d03763a6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -217,160 +217,154 @@ def run(self, context: ExecutionContext): self.end_test_scenario() def _attempt_plan_flight_conflict(self) -> OperationalIntentReference: + self.begin_test_step("Plan Flight 2") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 2") _, self.flight2_id, _ = plan_flight_intent( self, self.control_uss, self.flight2_planned.request, ) - self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2_planned.request) + self.end_test_step() + self.begin_test_step("Activate Flight 2") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, flight_2_oi_ref, ) as validator: activate_flight_intent( self, - "Activate Flight 2", self.control_uss, self.flight2_activated.request, self.flight2_id, ) flight_2_oi_ref = validator.expect_shared(self.flight2_activated.request) + self.end_test_step() + self.begin_test_step("Attempt to plan Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 not shared", self._intents_extent, ) as validator: - self.begin_test_step("Attempt to plan Flight 1") plan_conflict_flight_intent( self, self.tested_uss, self.flight1_planned.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() return flight_2_oi_ref def _attempt_activate_flight_conflict(self): + self.begin_test_step("Attempt to directly activate conflicting Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 not shared", self._intents_extent, ) as validator: - self.begin_test_step("Attempt to directly activate conflicting Flight 1") activate_conflict_flight_intent( self, self.tested_uss, self.flight1_activated.request, self.flight1_id, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() def _attempt_modify_planned_flight_conflict( self, ) -> Optional[OperationalIntentReference]: + self.begin_test_step("Plan Flight 1c") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1c sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 1c") _, self.flight1_id, _ = plan_flight_intent( self, self.tested_uss, self.flight1c_planned.request, ) - self.end_test_step() flight_1_oi_ref = validator.expect_shared(self.flight1c_planned.request) + self.end_test_step() + self.begin_test_step("Attempt to modify planned Flight 1c into conflict") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1c not modified", self._intents_extent, flight_1_oi_ref, ) as validator: - self.begin_test_step("Attempt to modify planned Flight 1c into conflict") modify_planned_conflict_flight_intent( self, self.tested_uss, self.flight1_planned.request, self.flight1_id, ) - self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1c_planned.request, skip_if_not_found=True ) + self.end_test_step() return flight_1_oi_ref def _attempt_modify_activated_flight_conflict( self, flight_1_oi_ref: Optional[OperationalIntentReference] ) -> Optional[OperationalIntentReference]: + self.begin_test_step("Activate Flight 1c") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1c sharing", self._intents_extent, flight_1_oi_ref, ) as validator: activate_flight_intent( self, - "Activate Flight 1c", self.tested_uss, self.flight1c_activated.request, self.flight1_id, ) flight_1_oi_ref = validator.expect_shared(self.flight1c_activated.request) + self.end_test_step() + self.begin_test_step("Attempt to modify activated Flight 1c into conflict") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1c not modified", self._intents_extent, flight_1_oi_ref, ) as validator: - self.begin_test_step("Attempt to modify activated Flight 1c into conflict") modify_activated_conflict_flight_intent( self, self.tested_uss, self.flight1_activated.request, self.flight1_id, ) - self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1c_activated.request, skip_if_not_found=True ) + self.end_test_step() - _ = delete_flight_intent( - self, "Delete Flight 2", self.control_uss, self.flight2_id - ) + self.begin_test_step("Delete Flight 2") + _ = delete_flight_intent(self, self.control_uss, self.flight2_id) self.flight2_id = None + self.end_test_step() return flight_1_oi_ref @@ -378,48 +372,46 @@ def _modify_activated_flight_preexisting_conflict( self, flight_1_oi_ref: Optional[OperationalIntentReference], ): + self.begin_test_step("Activate Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 sharing", self._intents_extent, flight_1_oi_ref, ) as validator: activate_flight_intent( self, - "Activate Flight 1", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) flight_1_oi_ref = validator.expect_shared(self.flight1_activated.request) + self.end_test_step() + self.begin_test_step("Plan Flight 2m") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2m sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 2m") _, self.flight2_id, _ = plan_flight_intent( self, self.control_uss, self.flight2m_planned.request, ) - self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2m_planned.request) + self.end_test_step() + self.begin_test_step("Declare Flight 2 non-conforming") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, flight_2_oi_ref, ) as validator: - self.begin_test_step("Declare Flight 2 non-conforming") resp_flight_2, _, _ = submit_flight_intent( self, "Successful transition to non-conforming state", @@ -432,7 +424,6 @@ def _modify_activated_flight_preexisting_conflict( self.flight2_nonconforming.request, self.flight2_id, ) - self.end_test_step() if resp_flight_2.result == InjectFlightResponseResult.NotSupported: msg = f"{self.control_uss.config.participant_id} does not support the transition to a Nonconforming state; execution of the scenario was stopped without failure" @@ -440,18 +431,18 @@ def _modify_activated_flight_preexisting_conflict( raise ScenarioCannotContinueError(msg) validator.expect_shared(self.flight2_nonconforming.request) + self.end_test_step() + self.begin_test_step( + "Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2" + ) with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1", self._intents_extent, flight_1_oi_ref, ) as validator: - self.begin_test_step( - "Attempt to modify activated Flight 1 in conflict with nonconforming Flight 2" - ) resp_flight_1, _, _ = submit_flight_intent( self, "Successful modification or rejection", @@ -464,7 +455,6 @@ def _modify_activated_flight_preexisting_conflict( self.flight1m_activated.request, self.flight1_id, ) - self.end_test_step() if resp_flight_1.result == InjectFlightResponseResult.ReadyToFly: validator.expect_shared(self.flight1m_activated.request) @@ -472,6 +462,7 @@ def _modify_activated_flight_preexisting_conflict( validator.expect_shared( self.flight1_activated.request, skip_if_not_found=True ) + self.end_test_step() def cleanup(self): self.begin_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md index 8469fbf59b..e18af04232 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md @@ -91,16 +91,20 @@ DSSInstanceResource that provides access to a DSS instance where flight creation ## Attempt to plan flight in conflict test case ![Test case summary illustration](assets/attempt_to_plan_flight_into_conflict.svg) -### [Plan Flight 2 test step](../../../../flight_planning/plan_flight_intent.md) +### Plan Flight 2 test step + +#### [Plan Flight 2](../../../../flight_planning/plan_flight_intent.md) The higher priority flight should be successfully planned by the control USS. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) + +### Attempt to plan Flight 1 test step -### [Attempt to plan Flight 1 test step](../../../../flight_planning/plan_priority_conflict_flight_intent.md) +#### [Attempt to plan Flight 1](../../../../flight_planning/plan_priority_conflict_flight_intent.md) The test driver attempts to plan the Flight 1 via the tested USS. However, it conflicts with Flight 2, which is of higher priority. As such it should be rejected per **[astm.f3548.v21.SCD0015](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1 not shared test step](../../validate_not_shared_operational_intent.md) +#### [Validate Flight 1 not shared](../../validate_not_shared_operational_intent.md) ### [Delete Flight 2 test step](../../../../flight_planning/delete_flight_intent.md) @@ -108,23 +112,29 @@ higher priority. As such it should be rejected per **[astm.f3548.v21.SCD0015](.. ## Attempt to modify planned flight in conflict test case ![Test case summary illustration](assets/attempt_to_modify_planned_flight_into_conflict.svg) -### [Plan Flight 1 test step](../../../../flight_planning/plan_flight_intent.md) +### Plan Flight 1 test step + +#### [Plan Flight 1](../../../../flight_planning/plan_flight_intent.md) The first flight should be successfully planned by the tested USS. -### [Validate Flight 1 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 sharing](../../validate_shared_operational_intent.md) + +### Plan Flight 2 test step -### [Plan Flight 2 test step](../../../../flight_planning/plan_flight_intent.md) +#### [Plan Flight 2](../../../../flight_planning/plan_flight_intent.md) The second flight should be successfully planned by the control USS. It conflicts with Flight 1, but it has higher priority. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) -### [Attempt to modify planned Flight 1 in conflict test step](../../../../flight_planning/modify_planned_priority_conflict_flight_intent.md) +### Attempt to modify planned Flight 1 in conflict test step + +#### [Attempt to modify Flight 1](../../../../flight_planning/modify_planned_priority_conflict_flight_intent.md) The test driver attempts to modify Flight 1 via the tested USS, which is planned. However, it conflicts with Flight 2, which is of higher priority and was planned in the meantime. As such it should be rejected per **[astm.f3548.v21.SCD0020](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1 not modified test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 not modified](../../validate_shared_operational_intent.md) Because the modification attempt was invalid, either Flight 1 should not have been modified (because the USS kept the original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). @@ -132,12 +142,14 @@ original accepted request), or it should have been removed (because the USS reje ## Attempt to activate flight in conflict test case ![Test case summary illustration](assets/attempt_to_activate_flight_into_conflict.svg) -### [Attempt to activate conflicting Flight 1 test step](../../../../flight_planning/activate_priority_conflict_flight_intent.md) +### Attempt to activate conflicting Flight 1 test step + +#### [Attempt to activate Flight 1](../../../../flight_planning/activate_priority_conflict_flight_intent.md) The test driver attempts to activate Flight 1, however, it conflicts with Flight 2, which is also planned and of higher priority. Note that Flight 1 could be either planned or non-existent before this step. As such it should be rejected per **[astm.f3548.v21.SCD0025](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1 not activated test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 not activated](../../validate_shared_operational_intent.md) Because the modification attempt was invalid, either Flight 1 should not have been modified (because the USS kept the original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). @@ -147,24 +159,32 @@ original accepted request), or it should have been removed (because the USS reje ### [Delete Flight 2 test step](../../../../flight_planning/delete_flight_intent.md) -### [Activate Flight 1 test step](../../../../flight_planning/activate_flight_intent.md) +### Activate Flight 1 test step + +#### [Activate Flight 1](../../../../flight_planning/activate_flight_intent.md) The test driver activates Flight 1, which should be done successfully given that it is now the highest-priority flight. Note that Flight 1 could be either planned or non-existent before this step. In the latter case, the flight will be directly activated without being planned beforehand. -### [Validate Flight 1 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 sharing](../../validate_shared_operational_intent.md) -### [Plan Flight 2 test step](../../../../flight_planning/plan_flight_intent.md) +### Plan Flight 2 test step + +#### [Plan Flight 2](../../../../flight_planning/plan_flight_intent.md) The second flight should be successfully planned by the control USS. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) + +### Activate Flight 2 test step -### [Activate Flight 2 test step](../../../../flight_planning/activate_flight_intent.md) +#### [Activate Flight 2](../../../../flight_planning/activate_flight_intent.md) The test driver activates Flight 2, which should be done successfully given that it is the highest-priority flight. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) -### [Modify activated Flight 1 in conflict with activated Flight 2 test step](../../../../flight_planning/modify_activated_flight_intent.md) +### Modify activated Flight 1 in conflict with activated Flight 2 test step + +#### [Modify Flight 1](../../../../flight_planning/modify_activated_flight_intent.md) Before execution of this step, flights 1 and 2 are activated and in conflict. Flight 2 is the highest-priority flight. The test driver attempts to modify Flight 1 in a way that still conflicts with Flight 2. @@ -184,7 +204,7 @@ In any case, whatever is the outcome of this step, there should not be any impac scenario. An intent should exist (this is checked in the next step) and it should be either the previous or the modified intent, both of which make no difference in the next steps. -### [Validate Flight 1 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 sharing](../../validate_shared_operational_intent.md) If the modification was accepted, Flight 1 should have been modified. If the modification was not supported, Flight 1 should not have been modified. If the modification was rejected, Flight 1 should not have been modified and should still exist. If it does not exist, @@ -194,24 +214,26 @@ it means that there is an active flight without an operational intent, which is ## Attempt to modify activated flight in conflict test case ![Test case summary illustration](assets/attempt_to_modify_activated_flight_into_conflict.svg) -### [Modify activated Flight 2 to not conflict with activated Flight 1 test step](../../../../flight_planning/modify_planned_flight_intent.md) +### Modify activated Flight 2 to not conflict with activated Flight 1 test step + +#### [Modify Flight 2](../../../../flight_planning/modify_planned_flight_intent.md) The test driver modifies (activated) Flight 2 with the control USS so that it is not anymore in conflict with (activated) flight of test USS. As Flight 2 is of higher priority, this should succeed and leave Flight 1 clear of conflict. -### [Validate Flight 2 sharing test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 2 sharing](../../validate_shared_operational_intent.md) -### [Attempt to modify activated Flight 1 in conflict test step](../../../../flight_planning/modify_activated_priority_conflict_flight_intent.md) +### Attempt to modify activated Flight 1 in conflict test step + +#### [Attempt to modify Flight 1](../../../../flight_planning/modify_activated_priority_conflict_flight_intent.md) The test driver attempts to modify Flight 1 so that it becomes in conflict with Flight 2. Both flights are activated at that point. However, because the conflict did not exist when the modification was initiated, it should be rejected per **[astm.f3548.v21.SCD0030](../../../../../requirements/astm/f3548/v21.md)**. -### [Validate Flight 1 not modified test step](../../validate_shared_operational_intent.md) +#### [Validate Flight 1 not modified](../../validate_shared_operational_intent.md) Because the modification attempt was invalid, either Flight 1 should not have been modified (because the USS kept the original accepted request), or it should have been removed (because the USS rejected the replacement plan provided). - - ## Cleanup ### ⚠️ Successful flight deletion check **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py index 1cbab012ac..3353f5cc5f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py @@ -212,194 +212,188 @@ def run(self, context: ExecutionContext): self.end_test_scenario() def _attempt_plan_flight_conflict(self): + self.begin_test_step("Plan Flight 2") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 2") resp_flight_2, self.flight2_id, _ = plan_flight_intent( self, self.control_uss, self.flight2_planned.request, ) - self.end_test_step() validator.expect_shared(self.flight2_planned.request) + self.end_test_step() + self.begin_test_step("Attempt to plan Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 not shared", self._intents_extent, ) as validator: - self.begin_test_step("Attempt to plan Flight 1") _ = plan_priority_conflict_flight_intent( self, self.tested_uss, self.flight1_planned.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() - _ = delete_flight_intent( - self, "Delete Flight 2", self.control_uss, self.flight2_id - ) + self.begin_test_step("Delete Flight 2") + _ = delete_flight_intent(self, self.control_uss, self.flight2_id) self.flight2_id = None + self.end_test_step() def _attempt_modify_planned_flight_conflict( self, ) -> Optional[OperationalIntentReference]: + self.begin_test_step("Plan Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 1") resp_flight_1, self.flight1_id, _ = plan_flight_intent( self, self.tested_uss, self.flight1_planned.request, ) - self.end_test_step() flight_1_oi_ref = validator.expect_shared(self.flight1_planned.request) + self.end_test_step() + self.begin_test_step("Plan Flight 2") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 2") resp_flight_2, self.flight2_id, _ = plan_flight_intent( self, self.control_uss, self.flight2_planned.request, ) - self.end_test_step() validator.expect_shared(self.flight2_planned.request) + self.end_test_step() + self.begin_test_step("Attempt to modify planned Flight 1 in conflict") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 not modified", self._intents_extent, flight_1_oi_ref, ) as validator: - self.begin_test_step("Attempt to modify planned Flight 1 in conflict") _ = modify_planned_priority_conflict_flight_intent( self, self.tested_uss, self.flight1m_planned.request, self.flight1_id, ) - self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1_planned.request, skip_if_not_found=True ) + self.end_test_step() return flight_1_oi_ref def _attempt_activate_flight_conflict( self, flight_1_oi_ref: Optional[OperationalIntentReference] ) -> Optional[OperationalIntentReference]: + self.begin_test_step("Attempt to activate conflicting Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 not activated", self._intents_extent, flight_1_oi_ref, ) as validator: - self.begin_test_step("Attempt to activate conflicting Flight 1") _ = activate_priority_conflict_flight_intent( self, self.tested_uss, self.flight1_activated.request, self.flight1_id, ) - self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1_planned.request, skip_if_not_found=True ) + self.end_test_step() return flight_1_oi_ref def _modify_activated_flight_conflict_preexisting( self, flight_1_oi_ref: Optional[OperationalIntentReference] ) -> Tuple[FlightIntent, OperationalIntentReference, OperationalIntentReference]: - _ = delete_flight_intent( - self, "Delete Flight 2", self.control_uss, self.flight2_id - ) + self.begin_test_step("Delete Flight 2") + _ = delete_flight_intent(self, self.control_uss, self.flight2_id) self.flight2_id = None + self.end_test_step() + self.begin_test_step("Activate Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 sharing", self._intents_extent, flight_1_oi_ref, ) as validator: activate_flight_intent( self, - "Activate Flight 1", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) flight_1_oi_ref = validator.expect_shared(self.flight1_activated.request) + self.end_test_step() + self.begin_test_step("Plan Flight 2") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, ) as validator: - self.begin_test_step("Plan Flight 2") _, self.flight2_id, _ = plan_flight_intent( self, self.control_uss, self.flight2_planned.request, ) - self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2_planned.request) + self.end_test_step() + self.begin_test_step("Activate Flight 2") with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, flight_2_oi_ref, ) as validator: activate_flight_intent( self, - "Activate Flight 2", self.control_uss, self.flight2_activated.request, self.flight2_id, ) flight_2_oi_ref = validator.expect_shared(self.flight2_activated.request) + self.end_test_step() + self.begin_test_step( + "Modify activated Flight 1 in conflict with activated Flight 2" + ) with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 sharing", self._intents_extent, flight_1_oi_ref, ) as validator: resp = modify_activated_flight_intent( self, - "Modify activated Flight 1 in conflict with activated Flight 2", self.tested_uss, self.flight1m_activated.request, self.flight1_id, @@ -410,12 +404,14 @@ def _modify_activated_flight_conflict_preexisting( flight_1_oi_ref = validator.expect_shared( self.flight1m_activated.request ) - return self.flight1m_activated, flight_1_oi_ref, flight_2_oi_ref + result = (self.flight1m_activated, flight_1_oi_ref, flight_2_oi_ref) else: flight_1_oi_ref = validator.expect_shared( self.flight1_activated.request ) - return self.flight1_activated, flight_1_oi_ref, flight_2_oi_ref + result = (self.flight1_activated, flight_1_oi_ref, flight_2_oi_ref) + self.end_test_step() + return result def _attempt_modify_activated_flight_conflict( self, @@ -423,43 +419,44 @@ def _attempt_modify_activated_flight_conflict( flight_1_oi_ref: OperationalIntentReference, flight_2_oi_ref: OperationalIntentReference, ): + self.begin_test_step( + "Modify activated Flight 2 to not conflict with activated Flight 1" + ) with OpIntentValidator( self, self.control_uss, self.dss, - "Validate Flight 2 sharing", self._intents_extent, flight_2_oi_ref, ) as validator: modify_activated_flight_intent( self, - "Modify activated Flight 2 to not conflict with activated Flight 1", self.control_uss, self.flight2m_activated.request, self.flight2_id, ) validator.expect_shared(self.flight2m_activated.request) + self.end_test_step() + self.begin_test_step("Attempt to modify activated Flight 1 in conflict") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 not modified", self._intents_extent, flight_1_oi_ref, ) as validator: - self.begin_test_step("Attempt to modify activated Flight 1 in conflict") modify_activated_priority_conflict_flight_intent( self, self.tested_uss, self.flight1c_activated.request, self.flight1_id, ) - self.end_test_step() validator.expect_shared( flight_1_intent.request, skip_if_not_found=True, ) + self.end_test_step() def cleanup(self): self.begin_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.md b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.md index 06a7da38b2..40c01a0fbf 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.md @@ -92,10 +92,6 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept Flight 1. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### Validate Flight 1 status test step -This step validates that the response of the USS is consistent with the flight shared, i.e. either it was properly -planned, or the USS rejected the planning. - #### [Validate accepted Flight 1 status](../validate_shared_operational_intent.md) If the planning was accepted, Flight 1 should have been shared. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py index f0bcac5e50..c386f79458 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss.py @@ -227,14 +227,13 @@ def _plan_flight_conflict_planned(self): self.end_test_step() # Tested USS attempts to plan Flight 1 test step + self.begin_test_step("Tested USS attempts to plan Flight 1") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate Flight 1 status", self._intents_extent, ) as validator: - self.begin_test_step("Tested USS attempts to plan Flight 1") resp, flight_id, _ = submit_flight_intent( self, "Successful planning", @@ -252,7 +251,6 @@ def _plan_flight_conflict_planned(self): ) if resp.result == InjectFlightResponseResult.Planned: - self.end_test_step() validator.expect_shared(self.flight1_planned.request) elif ( resp.result == InjectFlightResponseResult.Rejected @@ -272,8 +270,8 @@ def _plan_flight_conflict_planned(self): severity=Severity.Low, details=check_details, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() def _clear_op_intents(self): oi_refs, find_query = self.dss.find_op_intent(self._intents_extent) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md index 4bca58ad6f..0a62de8d92 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md @@ -89,7 +89,7 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept Flight 2. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate high-priority Flight 2 not shared test step](../validate_not_shared_operational_intent.md) +#### [Validate high-priority Flight 2 not shared](../validate_not_shared_operational_intent.md) ### [Restore virtual USS availability at DSS test step](../set_uss_available.md) @@ -126,7 +126,7 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept Flight 2. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate high-priority Flight 2 not shared test step](../validate_not_shared_operational_intent.md) +#### [Validate high-priority Flight 2 not shared](../validate_not_shared_operational_intent.md) ### [Restore virtual USS availability at DSS test step](../set_uss_available.md) @@ -163,7 +163,7 @@ All flight intent data provided was complete and correct. It should have been pr to reject or accept Flight 2. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. -### [Validate high-priority Flight 2 not shared test step](../validate_not_shared_operational_intent.md) +#### [Validate high-priority Flight 2 not shared](../validate_not_shared_operational_intent.md) ## Cleanup diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py index 315843982d..9da4eaee80 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py @@ -95,14 +95,13 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference: self.end_test_step() # Tested USS attempts to plan high-priority flight 2 test step + self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate high-priority Flight 2 not shared", self._intents_extent, ) as validator: - self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") submit_flight_intent( self, "Incorrectly planned", @@ -116,9 +115,9 @@ def _plan_flight_conflict_activated(self) -> OperationalIntentReference: self.tested_uss, self.flight2_planned.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() # Restore virtual USS availability at DSS test step self.begin_test_step("Restore virtual USS availability at DSS") @@ -142,14 +141,13 @@ def _plan_flight_conflict_nonconforming( self.end_test_step() # Tested USS attempts to plan high-priority flight 2 test step + self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate high-priority Flight 2 not shared", self._intents_extent, ) as validator: - self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") submit_flight_intent( self, "Incorrectly planned", @@ -163,9 +161,9 @@ def _plan_flight_conflict_nonconforming( self.tested_uss, self.flight2_planned.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() # Restore virtual USS availability at DSS test step self.begin_test_step("Restore virtual USS availability at DSS") @@ -187,14 +185,13 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference): self.end_test_step() # Tested USS attempts to plan high-priority flight 2 test step + self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") with OpIntentValidator( self, self.tested_uss, self.dss, - "Validate high-priority Flight 2 not shared", self._intents_extent, ) as validator: - self.begin_test_step("Tested USS attempts to plan high-priority Flight 2") submit_flight_intent( self, "Incorrectly planned", @@ -208,6 +205,6 @@ def _plan_flight_conflict_contingent(self, oi_ref: OperationalIntentReference): self.tested_uss, self.flight2_planned.request, ) - self.end_test_step() validator.expect_not_shared() + self.end_test_step() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index dd8acb98ae..d3edc1aa0f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -54,22 +54,19 @@ def __init__( scenario: TestScenarioType, flight_planner: Union[FlightPlanner, FlightPlannerClient], dss: DSSInstance, - test_step: str, extent: Volume4D, orig_oi_ref: Optional[OperationalIntentReference] = None, ): """ - :param scenario: - :param flight_planner: - :param dss: - :param test_step: + :param scenario: test scenario in which the operational intent is being validated. + :param flight_planner: flight planner responsible for maintenance of the operational intent. + :param dss: DSS instance in which to check for operational intents. :param extent: the extent over which the operational intents are to be compared. :param orig_oi_ref: if this is validating a previously existing operational intent (e.g. modification), pass the original reference. """ self._scenario: TestScenarioType = scenario self._flight_planner: Union[FlightPlanner, FlightPlannerClient] = flight_planner self._dss: DSSInstance = dss - self._test_step: str = test_step self._extent: Volume4D = extent self._orig_oi_ref: Optional[OperationalIntentReference] = orig_oi_ref @@ -91,7 +88,7 @@ def _find_after_oi(self, oi_id: str) -> Optional[OperationalIntentReference]: found = [oi_ref for oi_ref in self._after_oi_refs if oi_ref.id == oi_id] return found[0] if len(found) != 0 else None - def _begin_step(self): + def _begin_step_fragment(self): self._after_oi_refs, self._after_query = self._dss.find_op_intent(self._extent) oi_ids_delta = {oi_ref.id for oi_ref in self._after_oi_refs} - { oi_ref.id for oi_ref in self._before_oi_refs @@ -106,7 +103,6 @@ def _begin_step(self): if len(oi_ids_delta) == 1: self._new_oi_ref = self._find_after_oi(oi_ids_delta.pop()) - self._scenario.begin_test_step(self._test_step) self._scenario.record_query(self._before_query) self._scenario.record_query(self._after_query) @@ -126,12 +122,41 @@ def _begin_step(self): query_timestamps=[self._after_query.request.timestamp], ) + def expect_removed(self, oi_id: EntityID) -> None: + """Validate that a specific operational intent reference was removed from the DSS. + + It implements the test step described in validate_removed_operational_intent.md. + """ + self._begin_step_fragment() + + with self._scenario.check( + "Operational intent not shared", self._flight_planner.participant_id + ) as check: + if oi_id in [oi_ref.id for oi_ref in self._after_oi_refs]: + check.record_failed( + summary=f"Removed flight's op intent {oi_id} remained shared", + details=f"{self._flight_planner.participant_id} should have removed their flight which should include removal of the corresponding operational intent from the interoperability ecosystem, but a reference to operational intent {oi_id} was still found in the DSS after removal", + query_timestamps=[ + self._before_query.request.timestamp, + self._after_query.request.timestamp, + ], + ) + if self._new_oi_ref is not None: + check.record_failed( + summary=f"New op intent {self._new_oi_ref.id} was shared", + details=f"{self._flight_planner.participant_id} should have removed their flight which should have included removal of the corresponding operational intent from the interoperability ecosystem, but a new operational intent {self._new_oi_ref.id} was created during the time {oi_id} should have been removed", + query_timestamps=[ + self._before_query.request.timestamp, + self._after_query.request.timestamp, + ], + ) + def expect_not_shared(self) -> None: """Validate that an operational intent information was not shared with the DSS. It implements the test step described in validate_not_shared_operational_intent.md. """ - self._begin_step() + self._begin_step_fragment() with self._scenario.check( "Operational intent not shared", [self._flight_planner.participant_id] @@ -144,8 +169,6 @@ def expect_not_shared(self) -> None: query_timestamps=[self._after_query.request.timestamp], ) - self._scenario.end_test_step() - def expect_shared( self, flight_intent: Union[InjectFlightRequest, FlightInfo], @@ -163,10 +186,9 @@ def expect_shared( if isinstance(flight_intent, InjectFlightRequest): flight_intent = FlightInfo.from_scd_inject_flight_request(flight_intent) - self._begin_step() + self._begin_step_fragment() oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) if oi_ref is None: - self._scenario.end_test_step() return None self._check_op_intent_details(flight_intent, oi_ref) @@ -180,7 +202,6 @@ def expect_shared( ): self._check_op_intent_telemetry(oi_ref) - self._scenario.end_test_step() return oi_ref def expect_shared_with_invalid_data( @@ -204,10 +225,9 @@ def expect_shared_with_invalid_data( if isinstance(flight_intent, InjectFlightRequest): flight_intent = FlightInfo.from_scd_inject_flight_request(flight_intent) - self._begin_step() + self._begin_step_fragment() oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) if oi_ref is None: - self._scenario.end_test_step() return None goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( @@ -235,7 +255,6 @@ def expect_shared_with_invalid_data( query_timestamps=[oi_full_query.request.timestamp], ) - self._scenario.end_test_step() return oi_ref def _operational_intent_retrievable_check( @@ -275,8 +294,8 @@ def _operational_intent_shared_check( ) else: self._scenario.record_note( - f"{self._flight_planner.participant_id} skipped step", - f"No new operational intent was found in DSS, instructed to skip test step '{self._test_step}'.", + f"{self._flight_planner.participant_id} no op intent", + f"No new operational intent was found in DSS for test step '{self._scenario.current_step_name()}'.", ) return None oi_ref = self._new_oi_ref @@ -317,8 +336,8 @@ def _operational_intent_shared_check( ) else: self._scenario.record_note( - f"{self._flight_planner.participant_id} skipped step", - f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS, instructed to skip test step '{self._test_step}'.", + f"{self._flight_planner.participant_id} no op intent", + f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS for test step '{self._scenario.current_step_name()}'.", ) return None oi_ref = modified_oi_ref diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_removed_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_removed_operational_intent.md new file mode 100644 index 0000000000..85026491f1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_removed_operational_intent.md @@ -0,0 +1,15 @@ +# Validate operational intent removed test step fragment + +This step verifies that ending/removal/cancellation of a flight resulted in the operational intent reference being removed from the DSS. +It does so by querying the DSS for operational intents in the area of the flight before and after an attempted removal. +This assumes an area lock on the extent of the flight intent. + +See `OpIntentValidator.expect_removed()` in [test_steps.py](test_steps.py). + +## 🛑 DSS responses check + +**[astm.f3548.v21.DSS0005,2](../../../requirements/astm/f3548/v21.md)** + +## 🛑 Operational intent not shared check +If the operational intent reference for the flight is still found in the area of the flight intent, this check will fail per +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../requirements/interuss/automated_testing/flight_planning.md)**. diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 9ca0e6ea8e..62cfad5813 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -78,7 +78,6 @@ def plan_flight_intent( def activate_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -94,7 +93,6 @@ def activate_flight_intent( flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _, _ = submit_flight_intent( scenario, "Successful activation", @@ -105,13 +103,11 @@ def activate_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_planned_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -125,7 +121,6 @@ def modify_planned_flight_intent( """ expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _, _ = submit_flight_intent( scenario, "Successful modification", @@ -136,13 +131,11 @@ def modify_planned_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -160,7 +153,6 @@ def modify_activated_flight_intent( flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) if preexisting_conflict: resp, flight_id, _ = submit_flight_intent( scenario, @@ -212,7 +204,6 @@ def modify_activated_flight_intent( flight_id, ) - scenario.end_test_step() return resp @@ -290,7 +281,6 @@ def submit_flight_intent( def delete_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_id: str, ) -> DeleteFlightResponse: @@ -303,7 +293,6 @@ def delete_flight_intent( Returns: The deletion response. """ - scenario.begin_test_step(test_step) with scenario.check( "Successful deletion", [flight_planner.participant_id] ) as check: @@ -322,7 +311,6 @@ def delete_flight_intent( notes_suffix = f': "{resp.notes}"' if "notes" in resp and resp.notes else "" if resp.result == DeleteFlightResponseResult.Closed: - scenario.end_test_step() return resp else: check.record_failed(