From 6833bd103f3826c6d9872cfcac9bef0c6baf0e10 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 18 Dec 2023 04:24:55 +0000 Subject: [PATCH] Clean up get op data validation scenario --- .../get_op_data_validation.md | 68 +++--- .../get_op_data_validation.py | 216 ++++++++++-------- .../expected_interactions_test_steps.py | 10 - .../test_steps/invalid_op_test_steps.py | 2 - ...alidate_notification_operational_intent.md | 4 +- .../scenarios/flight_planning/test_steps.py | 21 +- .../suites/astm/utm/f3548_21.yaml | 6 +- 7 files changed, 156 insertions(+), 171 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 4fbb2cdfb4..0a281850ba 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -2,8 +2,8 @@ ## Description This test checks that the USS being tested validates the operational intents received as response to its GET request from another USS. -Control_uss which is a mock uss plans a nearby V-shaped operation, and provides the data that tested_uss GETs. -tested_uss validates the GET response from control_uss and accordingly plan its operation. +mock_uss which is a mock uss plans a nearby V-shaped operation, and provides the data that tested_uss GETs. +tested_uss validates the GET response from mock_uss and accordingly plan its operation. Notably the following requirements: - **[astm.f3548.v21.SCD0035](../../../../requirements/astm/f3548/v21.md)** @@ -18,7 +18,7 @@ There is an overlap in time and altitude of the two flights. - flight_1 - flight_2 -### control_uss +### mock_uss MockUSSResource that will be used for planning flights, controlling data shared for validation testing, and gathering interuss interactions from mock_uss. ### tested_uss @@ -29,74 +29,66 @@ DSSInstanceResource that provides access to a DSS instance where flight creation ## Successfully plan flight near an existing flight test case -### [Control_uss plans flight 2 test step](../../../flight_planning/plan_flight_intent.md) +### [mock_uss plans flight 2 test step](../../../flight_planning/plan_flight_intent.md) Flight 2 should be successfully planned by the control USS. ### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) Validate that flight 2 is planned -### Check for notification to tested_uss due to subscription in flight 2 area test step -In order to run this test scenario, we need tested_uss to trigger GET operational intent request to mock_uss. -But, if there is a subscription by tested_uss, that would trigger notification of flight 2 to tested_uss. -Some USSes will not make a GET request to control_uss for flight 2 details, while planning a nearby flight, -if they got a notification for flight2. Hence, if a USS didn't make a GET request, we will only fail it if didn't get -a notification, or else, a precondition for the test will not be met. Some USSes might make a GET request despite getting -a notification, but as it would not be clear whether invalid information through notification or GET request was used for planning, -the test will be not be continued. - -### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) +### [tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 which is of equal priority and came first. ### [Validate flight 1 sharing test step](../validate_shared_operational_intent.md) Validate flight 1 is planned. +### Check for notification to tested_uss due to subscription in flight 2 area test step +In the following test step, we want to assert that tested_uss must have retrieved operational intent details from +mock_uss via a GET request. This assertion is only valid, however, if tested_uss did not obtain the operational +intent details in a different way -- specifically, a notification due to a pre-existing subscription. In this test +step, we determine if tested_uss had a pre-existing subscription by checking if mock_uss sent a notification to +tested_uss. + ### [Validate flight2 GET interaction, if no notification test step](test_steps/validate_get_operational_intent.md) -Validate that tested_uss makes a GET request for obtaining details of flight 2 from control_uss. -In a previous step (Precondition - check tested_uss has no subscription in flight 2 area), we ensured that no notification of flight 2 was sent to tested_uss. -Hence, tested_uss will need to make a GET request to obtain flight 2 details. +This step is skipped if a notification to tested_uss was found in the previous step. -### [Validate flight1 Notification sent to Control_uss test step](test_steps/validate_notification_operational_intent.md) -Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. +### [Validate flight1 Notification sent to mock_uss test step](test_steps/validate_notification_operational_intent.md) +tested_uss notifies flight 1 to mock_uss, due to its subscription through flight 2. ### [Delete tested_uss flight test step](../../../flight_planning/delete_flight_intent.md) Teardown -### [Delete control_uss flight test step](../../../flight_planning/delete_flight_intent.md) +### [Delete mock_uss flight test step](../../../flight_planning/delete_flight_intent.md) Teardown ## Flight planning prevented due to invalid data sharing test case -### [Control_uss plans flight 2, sharing invalid operational intent data test step](../../../flight_planning/plan_flight_intent.md) -Flight 2 should be successfully planned by the control_uss. -The control_uss, which is mock_uss is instructed to share invalid data with other USS, for negative test. +### [mock_uss plans flight 2, sharing invalid operational intent data test step](../../../flight_planning/plan_flight_intent.md) +Flight 2 should be successfully planned by the mock_uss. +The mock_uss, which is mock_uss is instructed to share invalid data with other USS, for negative test. ### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. -### Check for notification to tested_uss due to subscription in flight 2 area test step -In order to run this test scenario, we need tested_uss to trigger GET operational intent request to mock_uss. -But, if there is a subscription by tested_uss, that would trigger notification of flight 2 to tested_uss. -Some USSes will not make a GET request to control_uss for flight 2 details, while planning a nearby flight, -if they got a notification for flight2. Hence, if a USS didn't make a GET request, we will only fail it if didn't get -a notification, or else, a precondition for the test will not be met. Some USSes might make a GET request despite getting -a notification, but as it would not be clear whether invalid information through notification or GET request was used for planning, -the test will be not be continued. - -### [Tested_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) +### [tested_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. ### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) Validate flight 1 is not shared with DSS, as plan failed. +### Check for notification to tested_uss due to subscription in flight 2 area test step +In the following test step, we want to assert that tested_uss must have retrieved operational intent details from +mock_uss via a GET request. This assertion is only valid, however, if tested_uss did not obtain the operational +intent details in a different way -- specifically, a notification due to a pre-existing subscription. In this test +step, we determine if tested_uss had a pre-existing subscription by checking if mock_uss sent a notification to +tested_uss. + ### [Validate flight2 GET interaction, if no notification test step](test_steps/validate_get_operational_intent.md) -Validate that tested_uss makes a GET request for obtaining details of flight 2 from control_uss. -In a previous step (Precondition - check tested_uss has no subscription in flight 2 area), we ensured that no notification of flight 2 was sent to tested_uss. -Hence, tested_uss will need to make a GET request to obtain flight 2 details. +This step is skipped if a notification to tested_uss was found in the previous step. -### [Validate flight 1 Notification not sent to Control_uss test step](test_steps/validate_no_notification_operational_intent.md) +### [Validate flight 1 Notification not sent to mock_uss test step](test_steps/validate_no_notification_operational_intent.md) -### [Delete Control_uss flight test step](../../../flight_planning/delete_flight_intent.md) +### [Delete mock_uss flight test step](../../../flight_planning/delete_flight_intent.md) Teardown ## Cleanup diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index 9e5a341c72..ebe9592e22 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -1,4 +1,9 @@ from typing import Optional, Dict + +from monitoring.monitorlib.clients.flight_planning.flight_info import ( + AirspaceUsageState, + UasState, +) from monitoring.monitorlib.clients.flight_planning.flight_info_template import ( FlightInfoTemplate, ) @@ -11,6 +16,13 @@ from monitoring.uss_qualifier.resources.flight_planning import ( FlightIntentsResource, ) +from monitoring.uss_qualifier.resources.flight_planning.flight_intent import ( + FlightIntent, +) +from monitoring.uss_qualifier.resources.flight_planning.flight_intent_validation import ( + ExpectedFlightIntent, + validate_flight_intent_templates, +) from monitoring.uss_qualifier.resources.flight_planning.flight_planners import ( FlightPlannerResource, ) @@ -53,21 +65,21 @@ class GetOpResponseDataValidationByUSS(TestScenario): flight_2: FlightInfoTemplate tested_uss_client: FlightPlannerClient - control_uss: MockUSSClient - control_uss_client: FlightPlannerClient + mock_uss: MockUSSClient + mock_uss_client: FlightPlannerClient dss: DSSInstance def __init__( self, tested_uss: FlightPlannerResource, - control_uss: MockUSSResource, + mock_uss: MockUSSResource, dss: DSSInstanceResource, flight_intents: Optional[FlightIntentsResource] = None, ): super().__init__() self.tested_uss_client = tested_uss.client - self.control_uss = control_uss.mock_uss - self.control_uss_client = control_uss.mock_uss.flight_planner + self.mock_uss = mock_uss.mock_uss + self.mock_uss_client = mock_uss.mock_uss.flight_planner self.dss = dss.dss if not flight_intents: @@ -78,40 +90,46 @@ def __init__( ) raise ScenarioCannotContinueError(msg) - _flight_intents = flight_intents.get_flight_intents() - - t_now = Time(arrow.utcnow().datetime) - times = { - TimeDuringTest.StartOfTestRun: t_now, - TimeDuringTest.StartOfScenario: t_now, - TimeDuringTest.TimeOfEvaluation: t_now, - } - extents = [] - for intent in _flight_intents.values(): - extents.append(intent.resolve(times).basic_information.area.bounding_volume) - self._intents_extent = Volume4DCollection(extents).bounding_volume.to_f3548v21() - + expected_flight_intents = [ + ExpectedFlightIntent( + "flight_1", + "Flight 1", + must_not_conflict_with=["Flight 2"], + f3548v21_priority_equal_to=["Flight 2"], + usage_state=AirspaceUsageState.Planned, + uas_state=UasState.Nominal, + # TODO: Must intersect bounding box of Flight 2 + ), + ExpectedFlightIntent( + "flight_2", + "Flight 2", + must_not_conflict_with=["Flight 1"], + f3548v21_priority_equal_to=["Flight 1"], + usage_state=AirspaceUsageState.Planned, + uas_state=UasState.Nominal, + # TODO: Must intersect bounding box of Flight 1 + ), + ] + + templates = flight_intents.get_flight_intents() try: - (self.flight_1, self.flight_2,) = ( - _flight_intents["flight_1"], - _flight_intents["flight_2"], - ) - - assert not self.flight_1.resolve( - times - ).basic_information.area.intersects_vol4s( - self.flight_2.resolve(times).basic_information.area - ), "flight_1 and flight_2 must not intersect" - - except KeyError as e: - raise ValueError( - f"`{self.me()}` TestScenario requirements for flight_intents not met: missing flight intent {e}" - ) - except AssertionError as e: + validate_flight_intent_templates(templates, expected_flight_intents) + except ValueError as e: raise ValueError( f"`{self.me()}` TestScenario requirements for flight_intents not met: {e}" ) + extents = [] + for efi in expected_flight_intents: + intent = FlightIntent.from_flight_info_template(templates[efi.intent_id]) + extents.extend(intent.request.operational_intent.volumes) + extents.extend(intent.request.operational_intent.off_nominal_volumes) + setattr(self, efi.intent_id, templates[efi.intent_id]) + + self._intents_extent = Volume4DCollection.from_interuss_scd_api( + extents + ).bounding_volume.to_f3548v21() + def run(self, context: ExecutionContext): times = { TimeDuringTest.StartOfTestRun: Time(context.start_time), @@ -125,54 +143,40 @@ def run(self, context: ExecutionContext): ) self.record_note( "Control USS", - f"{self.control_uss_client.participant_id}", + f"{self.mock_uss_client.participant_id}", ) self.begin_test_case("Successfully plan flight near an existing flight") - self._tested_uss_plans_deconflicted_flight_near_existing_flight(times) + self._plan_successfully_test_case(times) self.end_test_case() self.begin_test_case("Flight planning prevented due to invalid data sharing") - self._tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight( - times - ) + self._plan_unsuccessfully_test_case(times) self.end_test_case() self.end_test_scenario() - def _tested_uss_plans_deconflicted_flight_near_existing_flight( - self, times: Dict[TimeDuringTest, Time] - ): + def _plan_successfully_test_case(self, times: Dict[TimeDuringTest, Time]): times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_2 = self.flight_2.resolve(times) with OpIntentValidator( self, - self.control_uss_client, + self.mock_uss_client, self.dss, "Validate flight 2 sharing", self._intents_extent, ) as validator: - planning_time = Time(arrow.utcnow().datetime) + self.begin_test_step("mock_uss plans flight 2") _, self.flight_2_id = plan_flight( self, - "Control_uss plans flight 2", - self.control_uss_client, + self.mock_uss_client, flight_2, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared(flight_2) - self.begin_test_step( - "Check for notification to tested_uss due to subscription in flight 2 area" - ) - tested_uss_notified = check_any_notification( - self, - self.control_uss, - planning_time, - ) - self.end_test_step() - times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_1 = self.flight_1.resolve(times) @@ -183,47 +187,60 @@ def _tested_uss_plans_deconflicted_flight_near_existing_flight( "Validate flight 1 sharing", self._intents_extent, ) as validator: + self.begin_test_step("tested_uss plans flight 1") planning_time = Time(arrow.utcnow().datetime) plan_res, self.flight_1_id = plan_flight( self, - "Tested_uss plans flight 1", self.tested_uss_client, flight_1, ) + self.end_test_step() validator.expect_shared( flight_1, ) + + self.begin_test_step( + "Check for notification to tested_uss due to subscription in flight 2 area" + ) + tested_uss_notified = check_any_notification( + self, + self.mock_uss, + planning_time, + ) + self.end_test_step() + + self.begin_test_step("Validate flight2 GET interaction, if no notification") if not tested_uss_notified: expect_get_requests_to_mock_uss_when_no_notification( self, - self.control_uss, + self.mock_uss, planning_time, - self.control_uss.base_url, + self.mock_uss.base_url, flight_2_oi_ref.id, self.tested_uss_client.participant_id, - "Validate flight2 GET interaction, if no notification", ) + self.end_test_step() + self.begin_test_step("Validate flight1 Notification sent to mock_uss") expect_interuss_post_interactions( self, - self.control_uss, + self.mock_uss, planning_time, - self.control_uss.base_url, + self.mock_uss.base_url, self.tested_uss_client.participant_id, plan_res.queries[0].request.timestamp, - "Validate flight1 Notification sent to Control_uss", ) + self.end_test_step() - delete_flight( - self, "Delete tested_uss flight", self.tested_uss_client, self.flight_1_id - ) - delete_flight( - self, "Delete control_uss flight", self.control_uss_client, self.flight_2_id - ) + self.begin_test_step("Delete tested_uss flight") + delete_flight(self, self.tested_uss_client, self.flight_1_id) + self.end_test_step() - def _tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight( - self, times: Dict[TimeDuringTest, Time] - ): + self.begin_test_step("Delete mock_uss flight") + delete_flight(self, self.mock_uss_client, self.flight_2_id) + self.end_test_step() + + def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]): times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_info = self.flight_2.resolve(times) @@ -241,35 +258,27 @@ def _tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight( additional_fields = {"behavior": behavior} with OpIntentValidator( self, - self.control_uss_client, + self.mock_uss_client, self.dss, "Validate flight 2 shared operational intent with invalid data", self._intents_extent, ) as validator: - planning_time = Time(arrow.utcnow().datetime) + self.begin_test_step( + "mock_uss plans flight 2, sharing invalid operational intent data" + ) _, self.flight_2_id = plan_flight( self, - "Control_uss plans flight 2, sharing invalid operational intent data", - self.control_uss_client, + self.mock_uss_client, flight_info, additional_fields, ) + self.end_test_step() flight_2_oi_ref = validator.expect_shared_with_invalid_data( flight_info, validation_failure_type=OpIntentValidationFailureType.DataFormat, invalid_fields=[modify_field1, modify_field2], ) - self.begin_test_step( - "Check for notification to tested_uss due to subscription in flight 2 area" - ) - tested_uss_notified = check_any_notification( - self, - self.control_uss, - planning_time, - ) - self.end_test_step() - times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) flight_1 = self.flight_1.resolve(times) with OpIntentValidator( @@ -279,41 +288,52 @@ def _tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight( "Validate flight 1 not shared by tested_uss", self._intents_extent, ) as validator: + self.begin_test_step("tested_uss attempts to plan flight 1, expect failure") planning_time = Time(arrow.utcnow().datetime) _, self.flight_1_id = plan_flight_intent_expect_failed( self, - "Tested_uss attempts to plan flight 1, expect failure", self.tested_uss_client, flight_1, ) + self.end_test_step() validator.expect_not_shared() + self.begin_test_step( + "Check for notification to tested_uss due to subscription in flight 2 area" + ) + tested_uss_notified = check_any_notification( + self, + self.mock_uss, + planning_time, + ) + self.end_test_step() + if not tested_uss_notified: + self.begin_test_step("Validate flight2 GET interaction, if no notification") expect_get_requests_to_mock_uss_when_no_notification( self, - self.control_uss, + self.mock_uss, planning_time, - self.control_uss.base_url, + self.mock_uss.base_url, flight_2_oi_ref.id, self.tested_uss_client.participant_id, - "Validate flight2 GET interaction, if no notification", ) + self.end_test_step() + self.begin_test_step("Validate flight 1 Notification not sent to mock_uss") expect_no_interuss_post_interactions( self, - self.control_uss, + self.mock_uss, planning_time, self.tested_uss_client.participant_id, - "Validate flight 1 Notification not sent to Control_uss", ) + self.end_test_step() - delete_flight( - self, "Delete Control_uss flight", self.control_uss_client, self.flight_2_id - ) + self.begin_test_step("Delete mock_uss flight") + delete_flight(self, self.mock_uss_client, self.flight_2_id) + self.end_test_step() def cleanup(self): self.begin_cleanup() - cleanup_flights_fp_client( - self, (self.control_uss_client, self.tested_uss_client) - ), + cleanup_flights_fp_client(self, (self.mock_uss_client, self.tested_uss_client)), self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py index 0278488416..a790f822d0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py @@ -26,7 +26,6 @@ def expect_interuss_post_interactions( posted_to_url: str, participant_id: str, plan_request_time: datetime.datetime, - test_step: str, ): """ This step checks if a notification was sent to a subscribed USS, from time 'st' to now @@ -36,7 +35,6 @@ def expect_interuss_post_interactions( plan_request_time: timestamp of the flight plan query that would lead to sending notification """ - scenario.begin_test_step(test_step) # Check for 'notification found' will be done periodically by waiting for a duration till max_wait_time time_waited = 0 @@ -64,7 +62,6 @@ def expect_interuss_post_interactions( details=f"Notification to {posted_to_url} not sent even though DSS instructed the planning USS to notify due to subscription.", query_timestamps=[plan_request_time, query.request.timestamp], ) - scenario.end_test_step() def expect_no_interuss_post_interactions( @@ -72,15 +69,12 @@ def expect_no_interuss_post_interactions( mock_uss: MockUSSClient, st: StringBasedDateTime, participant_id: str, - test_step: str, ): """ This step checks no notification was sent to any USS as no DSS entity was created, from time 'st' to now Args: participant_id: id of the participant responsible to send the notification """ - scenario.begin_test_step(test_step) - # Wait for next MaxTimeToWaitForSubscriptionNotificationSeconds duration to capture any notification time.sleep(max_wait_time) interactions, query = _get_interuss_interactions_with_check( @@ -97,7 +91,6 @@ def expect_no_interuss_post_interactions( details=f"Notification was wrongly sent for an entity not created.", query_timestamps=[query.request.timestamp], ) - scenario.end_test_step() def expect_get_requests_to_mock_uss_when_no_notification( @@ -107,7 +100,6 @@ def expect_get_requests_to_mock_uss_when_no_notification( mock_uss_base_url: str, id: str, participant_id: str, - test_step: str, ): """ This step checks a GET request was made to mock_uss for an existing entity, from time 'st' to now @@ -117,7 +109,6 @@ def expect_get_requests_to_mock_uss_when_no_notification( participant_id: id of the participant responsible to send GET request """ - scenario.begin_test_step(test_step) interactions, query = _get_interuss_interactions_with_check(scenario, mock_uss, st) logger.debug(f"Checking for GET request to {mock_uss_base_url} for id {id}") get_requested = False @@ -137,7 +128,6 @@ def expect_get_requests_to_mock_uss_when_no_notification( details=f"No GET request received at {mock_uss_base_url} for {id}. A planning USS in the area should have sent a reques to get the intent details.", query_timestamps=[query.request.timestamp], ) - scenario.end_test_step() def _get_interuss_interactions_with_check( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py index ed65613779..0b043d4e96 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py @@ -16,7 +16,6 @@ def plan_flight_intent_expect_failed( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlannerClient, flight_intent: FlightInfo, ) -> Tuple[PlanningActivityResponse, Optional[str]]: @@ -30,7 +29,6 @@ def plan_flight_intent_expect_failed( return submit_flight( scenario, - test_step, "Plan should fail", {(PlanningActivityResult.Failed, FlightPlanStatus.NotPlanned)}, {}, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md index 48c83c8e34..e95c6b2a2a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md @@ -26,8 +26,8 @@ To make sure the test driver gives enough time for notifications to be received it marks the time to get interactions from mock_uss as - the time test driver initiates the plan. The sequence of events is - 1. Test driver initiates plan to tested_uss. t0 -2. Tested_uss shares the plan with DSS and receives DSS response. t_time_start. -3. Tested_uss responds to test driver with Completed. +2. tested_uss shares the plan with DSS and receives DSS response. t_time_start. +3. tested_uss responds to test driver with Completed. 4. Test driver checks for shared operational_intent in DSS and checks its retrievable. t1 5. Test driver waits for MaxTimeToWaitForSubscriptionNotificationSeconds. 6. Test driver retrieves interactions from mock_uss. t1 + MaxTimeToWaitForSubscriptionNotificationSeconds diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 08d3f40976..5324b63f2a 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -388,14 +388,13 @@ def cleanup_flights( def plan_flight( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlannerClient, flight_info: FlightInfo, additional_fields: Optional[dict] = None, ) -> Tuple[PlanningActivityResponse, Optional[str]]: """Plan a flight intent that should result in success. - This function implements the test step described in + This function implements the test step fragment described in plan_flight_intent.md. Returns: @@ -404,7 +403,6 @@ def plan_flight( """ return submit_flight( scenario=scenario, - test_step=test_step, success_check="Successful planning", expected_results={(PlanningActivityResult.Completed, FlightPlanStatus.Planned)}, failed_checks={PlanningActivityResult.Failed: "Failure"}, @@ -416,10 +414,9 @@ def plan_flight( def submit_flight( scenario: TestScenarioType, - test_step: str, success_check: str, expected_results: Set[Tuple[PlanningActivityResult, FlightPlanStatus]], - failed_checks: Dict[PlanningActivityResult, Union[str, Tuple[str, Severity]]], + failed_checks: Dict[PlanningActivityResult, str], flight_planner: FlightPlannerClient, flight_info: FlightInfo, flight_id: Optional[str] = None, @@ -436,7 +433,6 @@ def submit_flight( * The ID of the injected flight if it is returned, None otherwise. """ - scenario.begin_test_step(test_step) with scenario.check(success_check, [flight_planner.participant_id]) as check: try: resp, query, flight_id = request_flight( @@ -447,7 +443,6 @@ def submit_flight( scenario.record_query(q) check.record_failed( summary=f"Error from {flight_planner.participant_id} when attempting to submit a flight intent (flight ID: {flight_id})", - severity=Severity.High, details=f"{str(e)}\n\nStack trace:\n{e.stacktrace}", query_timestamps=[q.request.timestamp for q in e.queries], ) @@ -455,11 +450,7 @@ def submit_flight( notes_suffix = f': "{resp.notes}"' if "notes" in resp and resp.notes else "" for unexpected_result, failed_test_check in failed_checks.items(): - if isinstance(failed_test_check, str): - check_name = failed_test_check - check_severity = Severity.High - else: - check_name, check_severity = failed_test_check + check_name = failed_test_check with scenario.check( check_name, [flight_planner.participant_id] @@ -467,18 +458,15 @@ def submit_flight( if resp.activity_result == unexpected_result: specific_failed_check.record_failed( summary=f"Flight unexpectedly {resp.activity_result}", - severity=check_severity, details=f'{flight_planner.participant_id} indicated {resp.activity_result} rather than the expected {" or ".join(r[0] for r in expected_results)}{notes_suffix}', query_timestamps=[query.request.timestamp], ) if (resp.activity_result, resp.flight_plan_status) in expected_results: - scenario.end_test_step() return resp, flight_id else: check.record_failed( summary=f"Flight planning activity outcome was not expected", - severity=Severity.High, details=f'{flight_planner.participant_id} indicated {resp.activity_result} with flight plan status {resp.flight_plan_status} rather than the expected {" or ".join([f"({expected_result[0]}, {expected_result[1]})" for expected_result in expected_results])}{notes_suffix}', query_timestamps=[query.request.timestamp], ) @@ -539,7 +527,6 @@ def cleanup_flight( def delete_flight( scenario: TestScenarioType, - test_step: str, flight_planner: FlightPlannerClient, flight_id: str, ) -> PlanningActivityResponse: @@ -550,7 +537,6 @@ def delete_flight( Returns: The deletion response. """ - scenario.begin_test_step(test_step) with scenario.check( "Successful deletion", [flight_planner.participant_id] ) as check: @@ -572,7 +558,6 @@ def delete_flight( resp.activity_result == PlanningActivityResult.Completed and resp.flight_plan_status == FlightPlanStatus.Closed ): - scenario.end_test_step() return resp else: check.record_failed( diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index 96fab43374..a68c1dfa36 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -124,14 +124,14 @@ actions: test_scenario: scenario_type: scenarios.astm.utm.data_exchange_validation.GetOpResponseDataValidationByUSS resources: - tested_uss: uss1 - control_uss: mock_uss + tested_uss: tested_uss + mock_uss: mock_uss dss: dss flight_intents: non_conflicting_flights on_failure: Continue flight_planners_source: flight_planners roles: - - uss1 + - tested_uss on_failure: Continue - action_generator: generator_type: action_generators.flight_planning.FlightPlannerCombinations