diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index aaa242d3c9..62226e5802 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -196,12 +196,13 @@ def _validate_ended_cancellation(self): "Validate flight intent shared correctly", self._intents_extent, ) as planned_validator: + self.begin_test_step("Plan flight intent") _, flight_id = plan_flight_intent( self, - "Plan flight intent", self.tested_uss, self.valid_flight.request, ) + self.end_test_step() planned_validator.expect_shared(self.valid_flight.request) _ = delete_flight_intent( @@ -210,12 +211,13 @@ def _validate_ended_cancellation(self): cancelled_validator.expect_not_shared() def _validate_precision_intersection(self): + self.begin_test_step("Plan control flight intent") _, _ = plan_flight_intent( self, - "Plan control flight intent", self.tested_uss, self.valid_flight.request, ) + self.end_test_step() with OpIntentValidator( self, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index 4e24bbabbb..d7b880f4aa 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -214,12 +214,13 @@ def _attempt_plan_flight_conflict(self) -> OperationalIntentReference: "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") _, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -246,12 +247,13 @@ def _attempt_plan_flight_conflict(self) -> OperationalIntentReference: "Validate Flight 1 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to plan Flight 1") plan_conflict_flight_intent( self, - "Attempt to plan Flight 1", self.tested_uss, self.flight1_planned.request, ) + self.end_test_step() validator.expect_not_shared() return flight_2_oi_ref @@ -264,13 +266,14 @@ def _attempt_activate_flight_conflict(self): "Validate Flight 1 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to directly activate conflicting Flight 1") activate_conflict_flight_intent( self, - "Attempt to directly activate conflicting Flight 1", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) + self.end_test_step() validator.expect_not_shared() def _attempt_modify_planned_flight_conflict( @@ -283,12 +286,13 @@ def _attempt_modify_planned_flight_conflict( "Validate Flight 1c sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 1c") _, self.flight1_id = plan_flight_intent( self, - "Plan Flight 1c", self.tested_uss, self.flight1c_planned.request, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared(self.flight1c_planned.request) with OpIntentValidator( @@ -299,13 +303,14 @@ def _attempt_modify_planned_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify planned Flight 1c into conflict") modify_planned_conflict_flight_intent( self, - "Attempt to modify planned Flight 1c into conflict", self.tested_uss, self.flight1_planned.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1c_planned.request, skip_if_not_found=True ) @@ -340,13 +345,14 @@ def _attempt_modify_activated_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify activated Flight 1c into conflict") modify_activated_conflict_flight_intent( self, - "Attempt to modify activated Flight 1c into conflict", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1c_activated.request, skip_if_not_found=True ) @@ -386,12 +392,13 @@ def _modify_activated_flight_preexisting_conflict( "Validate Flight 2m sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2m") _, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2m", self.control_uss, self.flight2m_planned.request, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2m_planned.request) with OpIntentValidator( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py index 32fd8e6ad6..3628c0988b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py @@ -214,12 +214,13 @@ def _attempt_plan_flight_conflict(self): "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") resp_flight_2, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -229,12 +230,13 @@ def _attempt_plan_flight_conflict(self): "Validate Flight 1 not shared", self._intents_extent, ) as validator: + self.begin_test_step("Attempt to plan Flight 1") _ = plan_priority_conflict_flight_intent( self, - "Attempt to plan Flight 1", self.tested_uss, self.flight1_planned.request, ) + self.end_test_step() validator.expect_not_shared() _ = delete_flight_intent( @@ -252,12 +254,13 @@ def _attempt_modify_planned_flight_conflict( "Validate Flight 1 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 1") resp_flight_1, self.flight1_id = plan_flight_intent( self, - "Plan Flight 1", self.tested_uss, self.flight1_planned.request, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared(self.flight1_planned.request) with OpIntentValidator( @@ -267,12 +270,13 @@ def _attempt_modify_planned_flight_conflict( "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") resp_flight_2, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -283,13 +287,14 @@ def _attempt_modify_planned_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify planned Flight 1 in conflict") _ = modify_planned_priority_conflict_flight_intent( self, - "Attempt to modify planned Flight 1 in conflict", self.tested_uss, self.flight1m_planned.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1_planned.request, skip_if_not_found=True ) @@ -307,13 +312,14 @@ def _attempt_activate_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to activate conflicting Flight 1") _ = activate_priority_conflict_flight_intent( self, - "Attempt to activate conflicting Flight 1", self.tested_uss, self.flight1_activated.request, self.flight1_id, ) + self.end_test_step() flight_1_oi_ref = validator.expect_shared( self.flight1_planned.request, skip_if_not_found=True ) @@ -352,12 +358,13 @@ def _modify_activated_flight_conflict_preexisting( "Validate Flight 2 sharing", self._intents_extent, ) as validator: + self.begin_test_step("Plan Flight 2") _, self.flight2_id = plan_flight_intent( self, - "Plan Flight 2", self.control_uss, self.flight2_planned.request, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(self.flight2_planned.request) with OpIntentValidator( @@ -436,13 +443,14 @@ def _attempt_modify_activated_flight_conflict( self._intents_extent, flight_1_oi_ref, ) as validator: + self.begin_test_step("Attempt to modify activated Flight 1 in conflict") modify_activated_priority_conflict_flight_intent( self, - "Attempt to modify activated Flight 1 in conflict", self.tested_uss, self.flight1c_activated.request, self.flight1_id, ) + self.end_test_step() validator.expect_shared( flight_1_intent.request, skip_if_not_found=True, diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py index 52f2357930..253e2dc117 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py @@ -19,7 +19,6 @@ def plan_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> InjectFlightResponse: @@ -30,11 +29,8 @@ def plan_priority_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly planned", @@ -47,13 +43,11 @@ def plan_priority_conflict_flight_intent( flight_intent, ) - scenario.end_test_step() return resp def modify_planned_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -65,11 +59,8 @@ def modify_planned_priority_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -83,13 +74,11 @@ def modify_planned_priority_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def activate_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -102,10 +91,9 @@ def activate_priority_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly activated", @@ -119,13 +107,11 @@ def activate_priority_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_priority_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -138,10 +124,9 @@ def modify_activated_priority_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -155,13 +140,11 @@ def modify_activated_priority_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def plan_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> InjectFlightResponse: @@ -172,11 +155,8 @@ def plan_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly planned", @@ -189,13 +169,11 @@ def plan_conflict_flight_intent( flight_intent, ) - scenario.end_test_step() return resp def modify_planned_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -207,11 +185,8 @@ def modify_planned_conflict_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -225,13 +200,11 @@ def modify_planned_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def activate_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -244,10 +217,9 @@ def activate_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly activated", @@ -261,13 +233,11 @@ def activate_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -280,10 +250,9 @@ def modify_activated_conflict_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Incorrectly modified", @@ -297,13 +266,11 @@ def modify_activated_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def plan_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> Tuple[InjectFlightResponse, Optional[str]]: @@ -312,15 +279,14 @@ def plan_permitted_conflict_flight_intent( This function implements the test step described in plan_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0055. + TODO: Remove this function if it is not used in the future + Returns: * The injection response. * The ID of the injected flight if it is returned, None otherwise. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, flight_id = submit_flight_intent( scenario, "Successful planning", @@ -330,13 +296,11 @@ def plan_permitted_conflict_flight_intent( flight_intent, ) - scenario.end_test_step() return resp, flight_id def modify_planned_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -346,13 +310,12 @@ def modify_planned_permitted_conflict_flight_intent( This function implements the test step described in modify_planned_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0060. + TODO: Remove this function if it is not used in the future + Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Successful modification", @@ -363,13 +326,11 @@ def modify_planned_permitted_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def activate_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: Optional[str] = None, @@ -379,13 +340,14 @@ def activate_permitted_conflict_flight_intent( This function implements the test step described in activate_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0065. + TODO: Remove this function if it is not used in the future + Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Successful activation", @@ -396,13 +358,11 @@ def activate_permitted_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp def modify_activated_permitted_conflict_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, flight_id: str, @@ -412,13 +372,14 @@ def modify_activated_permitted_conflict_flight_intent( This function implements the test step described in modify_activated_permitted_conflict_flight_intent.md. It validates requirement astm.f3548.v21.SCD0070. + TODO: Remove this function if it is not used in the future + Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) - scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( scenario, "Successful modification", @@ -429,5 +390,4 @@ def modify_activated_permitted_conflict_flight_intent( flight_id, ) - scenario.end_test_step() return resp diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 08d3f40976..385eebf91a 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -36,11 +36,11 @@ def expect_flight_intent_state( flight_intent: InjectFlightRequest, expected_state: OperationalIntentState, scenario: TestScenarioType, - test_step: str, ) -> None: """Confirm that provided flight intent test data has the expected state or raise a ValueError.""" if flight_intent.operational_intent.state != expected_state: function_name = str(inspect.stack()[1][3]) + test_step = scenario.current_step_name() raise ValueError( f"Error in test data: operational intent state for {function_name} during test step '{test_step}' in scenario '{scenario.documentation.name}' is expected to be `{expected_state}`, but got `{flight_intent.operational_intent.state}` instead" ) @@ -48,7 +48,6 @@ def expect_flight_intent_state( def plan_flight_intent( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlanner, flight_intent: InjectFlightRequest, ) -> Tuple[InjectFlightResponse, Optional[str]]: @@ -62,11 +61,8 @@ def plan_flight_intent( * The injection response. * The ID of the injected flight if it is returned, None otherwise. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) - scenario.begin_test_step(test_step) resp, flight_id = submit_flight_intent( scenario, "Successful planning", @@ -76,7 +72,6 @@ def plan_flight_intent( flight_intent, ) - scenario.end_test_step() return resp, flight_id @@ -95,7 +90,7 @@ def activate_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) scenario.begin_test_step(test_step) @@ -127,9 +122,7 @@ def modify_planned_flight_intent( Returns: The injection response. """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) + expect_flight_intent_state(flight_intent, OperationalIntentState.Accepted, scenario) scenario.begin_test_step(test_step) resp, _ = submit_flight_intent( @@ -163,7 +156,7 @@ def modify_activated_flight_intent( Returns: The injection response. """ expect_flight_intent_state( - flight_intent, OperationalIntentState.Activated, scenario, test_step + flight_intent, OperationalIntentState.Activated, scenario ) scenario.begin_test_step(test_step) diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index f95fca11ca..ccdec94961 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -242,6 +242,12 @@ def cleanup(self): def me(self) -> str: return inspection.fullname(self.__class__) + def current_step_name(self) -> Optional[str]: + if self._current_step: + return self._current_step.name + else: + return None + def _make_scenario_report(self) -> None: self._scenario_report = TestScenarioReport( name=self.documentation.name, diff --git a/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py b/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py index 4a20805057..2c1e1182cb 100644 --- a/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py +++ b/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py @@ -78,8 +78,10 @@ def run(self, context: ExecutionContext): self.end_test_case() self.begin_test_case("Plan valid flight") + self.begin_test_step("Plan valid flight intent") if not self._plan_valid_flight(): return + self.end_test_step() self.end_test_case() self.end_test_scenario() @@ -135,7 +137,6 @@ def _attempt_invalid_flights(self) -> bool: def _plan_valid_flight(self) -> bool: resp, _ = plan_flight_intent( self, - "Plan valid flight intent", self.ussp, self.valid_flight_intent.request, )