From e6cf0f96c6080c64645d322ad9d3f807ca568fe1 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Fri, 17 Nov 2023 01:13:05 -0800 Subject: [PATCH] Fixing per PR review comments --- .../clients/flight_planning/client.py | 8 +- .../clients/flight_planning/client_scd.py | 5 +- .../clients/flight_planning/client_v1.py | 9 +- .../resources/interuss/mock_uss/client.py | 11 +- .../get_op_data_validation.md | 2 +- .../get_op_data_validation.py | 239 +++++++++++++++++- .../test_steps/invalid_op_test_steps.py | 221 ++++++++++++++++ .../scenarios/astm/utm/test_steps.py | 6 +- .../scenarios/flight_planning/test_steps.py | 93 +++---- 9 files changed, 507 insertions(+), 87 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py diff --git a/monitoring/monitorlib/clients/flight_planning/client.py b/monitoring/monitorlib/clients/flight_planning/client.py index 2e670bee66..babf2b2b5b 100644 --- a/monitoring/monitorlib/clients/flight_planning/client.py +++ b/monitoring/monitorlib/clients/flight_planning/client.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Optional, Union +from typing import Optional, Set from monitoring.monitorlib.clients.flight_planning.test_preparation import ( TestPreparationActivityResponse, @@ -15,6 +15,7 @@ ) from monitoring.monitorlib.fetch import QueryError from monitoring.monitorlib.geotemporal import Volume4D +from monitoring.uss_qualifier.configurations.configuration import ParticipantID class PlanningActivityError(QueryError): @@ -24,6 +25,11 @@ class PlanningActivityError(QueryError): class FlightPlannerClient(ABC): """Client to interact with a USS as a user performing flight planning activities and as the test director preparing for tests involving flight planning activities.""" + def __init__(self, participant_id: ParticipantID): + self.participant_id = participant_id + self.created_flight_ids: Set[str] = set() + super(FlightPlannerClient, self).__init__() + # ===== Emulation of user actions ===== @abstractmethod diff --git a/monitoring/monitorlib/clients/flight_planning/client_scd.py b/monitoring/monitorlib/clients/flight_planning/client_scd.py index f19c7eb5c9..a9303dc0c0 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_scd.py +++ b/monitoring/monitorlib/clients/flight_planning/client_scd.py @@ -26,14 +26,17 @@ from monitoring.monitorlib.fetch import query_and_describe from monitoring.monitorlib.geotemporal import Volume4D from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.uss_qualifier.configurations.configuration import ParticipantID class SCDFlightPlannerClient(FlightPlannerClient): SCD_SCOPE = scd_api_constants.Scope.Inject _session: UTMClientSession _plan_statuses: Dict[FlightID, FlightPlanStatus] + participant_id: ParticipantID - def __init__(self, session: UTMClientSession): + def __init__(self, session: UTMClientSession, participant_id: ParticipantID): + super(SCDFlightPlannerClient, self).__init__(participant_id=participant_id) self._session = session self._plan_statuses = {} diff --git a/monitoring/monitorlib/clients/flight_planning/client_v1.py b/monitoring/monitorlib/clients/flight_planning/client_v1.py index 827a795a77..c5be7ed140 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_v1.py +++ b/monitoring/monitorlib/clients/flight_planning/client_v1.py @@ -1,6 +1,7 @@ import uuid -from typing import Optional, Set +from typing import Optional from implicitdict import ImplicitDict +from loguru import logger from monitoring.monitorlib.clients.flight_planning.client import ( FlightPlannerClient, ) @@ -31,12 +32,10 @@ class V1FlightPlannerClient(FlightPlannerClient): _session: UTMClientSession participant_id: ParticipantID - base_url: str def __init__(self, session: UTMClientSession, participant_id: ParticipantID): + super(V1FlightPlannerClient, self).__init__(participant_id=participant_id) self._session = session - self.participant_id = participant_id - self.created_flight_ids: Set[str] = set() def _inject( self, @@ -221,7 +220,7 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: raise PlanningActivityError( f"Response to clear area could not be parsed: {str(e)}", query ) - + self.created_flight_ids.clear() if resp.outcome.success: errors = None else: diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py b/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py index d10e2eecfb..15c192d72e 100644 --- a/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py +++ b/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py @@ -12,7 +12,7 @@ GetLocalityResponse, PutLocalityRequest, ) -from monitoring.monitorlib.fetch import QueryError, Query +from monitoring.monitorlib.fetch import QueryError from monitoring.monitorlib.infrastructure import AuthAdapter, UTMClientSession from monitoring.monitorlib.locality import LocalityCode from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( @@ -47,8 +47,9 @@ def __init__( self.session = UTMClientSession(base_url, auth_adapter, timeout_seconds) self.participant_id = participant_id v1_base_url = base_url + "/flight_planning/v1" - self.session_fp = UTMClientSession(v1_base_url, auth_adapter, timeout_seconds) - self.flight_planner = V1FlightPlannerClient(self.session_fp, participant_id) + self.flight_planner = V1FlightPlannerClient( + UTMClientSession(v1_base_url, auth_adapter, timeout_seconds), participant_id + ) def get_status(self) -> fetch.Query: return fetch.query_and_describe( @@ -86,7 +87,9 @@ def set_locality(self, locality_code: LocalityCode) -> fetch.Query: # TODO: Add other methods to interact with the mock USS in other ways (like starting/stopping message signing data collection) - def get_interactions(self, from_time: StringBasedDateTime) -> List[Interaction]: + def get_interactions( + self, from_time: StringBasedDateTime + ) -> Tuple[List[Interaction], fetch.Query]: """ Requesting interuss interactions from mock_uss from a given time till now Args: diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 3277dd0075..f6c3c67eb8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -78,7 +78,7 @@ No notification pushed by control_uss to tested_uss, will ensure that tested_uss while planning a nearby flight. If a notification is sent to tested_uss, the precondition for running this scenario will not be satisfied. -### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) +### [Tested_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index 6fa47e5464..740461d919 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -1,6 +1,7 @@ from typing import Optional -from loguru import logger +from urllib.parse import urlsplit +from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance from monitoring.uss_qualifier.resources.flight_planning import ( @@ -9,30 +10,48 @@ from monitoring.uss_qualifier.resources.flight_planning.flight_intent import ( FlightIntent, ) -from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( - FlightPlanner, -) from monitoring.uss_qualifier.resources.flight_planning.flight_planners import ( FlightPlannerResource, ) - -from monitoring.uss_qualifier.scenarios.scenario import ( - TestScenario, -) - +from monitoring.monitorlib.geotemporal import Volume4DCollection +from monitoring.monitorlib.clients.flight_planning.flight_info import FlightInfo from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( MockUSSClient, MockUSSResource, ) +from monitoring.uss_qualifier.scenarios.astm.utm.data_exchange_validation.test_steps.invalid_op_test_steps import ( + InvalidOpIntentSharingValidator, + plan_flight_intent_expect_failed, +) +from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import OpIntentValidator +from monitoring.uss_qualifier.scenarios.astm.utm.data_exchange_validation.test_steps.expected_interactions_test_steps import ( + expect_interuss_post_interactions, + expect_interuss_get_interactions, + expect_no_interuss_post_interactions, +) +from monitoring.monitorlib.clients.mock_uss.mock_uss_scd_injection_api import ( + MockUssFlightBehavior, +) +from monitoring.uss_qualifier.scenarios.scenario import ( + TestScenario, + ScenarioCannotContinueError, +) +from monitoring.uss_qualifier.scenarios.flight_planning.test_steps import ( + cleanup_flights_fp_client, + plan_flight, + delete_flight, +) +from implicitdict import StringBasedDateTime +from datetime import datetime class GetOpResponseDataValidationByUSS(TestScenario): flight_1: FlightIntent - flight_2: FlightIntent - tested_uss: FlightPlanner + tested_uss_client: FlightPlannerClient control_uss: MockUSSClient + control_uss_client: FlightPlannerClient dss: DSSInstance def __init__( @@ -43,16 +62,208 @@ def __init__( flight_intents: Optional[FlightIntentsResource] = None, ): super().__init__() - self.tested_uss = tested_uss.flight_planner + self.tested_uss_client = tested_uss.client self.control_uss = control_uss.mock_uss + self.control_uss_client = control_uss.mock_uss.flight_planner self.dss = dss.dss + if not flight_intents: + msg = f"No FlightIntentsResource was provided as input to this test, it is assumed that the jurisdiction of the tested USS ({self.tested_uss.config.participant_id}) does not allow any same priority conflicts, execution of the scenario was stopped without failure" + self.record_note( + "Jurisdiction of tested USS does not allow any same priority conflicts", + msg, + ) + raise ScenarioCannotContinueError(msg) + + _flight_intents = { + k: FlightIntent.from_flight_info_template(v) + for k, v in flight_intents.get_flight_intents().items() + } + + extents = [] + for intent in _flight_intents.values(): + extents.extend(intent.request.operational_intent.volumes) + extents.extend(intent.request.operational_intent.off_nominal_volumes) + self._intents_extent = Volume4DCollection.from_interuss_scd_api( + extents + ).bounding_volume.to_f3548v21() + + try: + (self.flight_1, self.flight_2,) = ( + _flight_intents["flight_1"], + _flight_intents["flight_2"], + ) + + assert not Volume4DCollection.from_interuss_scd_api( + self.flight_1.request.operational_intent.volumes + ).intersects_vol4s( + Volume4DCollection.from_interuss_scd_api( + self.flight_2.request.operational_intent.volumes + ) + ), "flight_1 and flight_2 must not intersect" + + except KeyError as e: + raise ValueError( + f"`{self.me()}` TestScenario requirements for flight_intents not met: missing flight intent {e}" + ) + except AssertionError as e: + raise ValueError( + f"`{self.me()}` TestScenario requirements for flight_intents not met: {e}" + ) + def run(self, context): self.begin_test_scenario(context) - pass + + self.record_note( + "Tested USS", + f"{self.tested_uss_client.participant_id}", + ) + self.record_note( + "Control USS", + f"{self.control_uss_client.participant_id}", + ) + + self.begin_test_case("Successfully plan flight near an existing flight") + self._tested_uss_plans_deconflicted_flight_near_existing_flight() + self.end_test_case() + + self.begin_test_case("Flight planning prevented due to invalid data sharing") + self._tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight() + self.end_test_case() + self.end_test_scenario() + def _tested_uss_plans_deconflicted_flight_near_existing_flight(self): + + with OpIntentValidator( + self, + self.control_uss_client, + self.dss, + "Validate flight 2 sharing", + self._intents_extent, + ) as validator: + _, self.flight_2_id = plan_flight( + self, + "Control_uss plans flight 2", + self.control_uss_client, + FlightInfo.from_scd_inject_flight_request(self.flight_2.request), + ) + + flight_2_oi_ref = validator.expect_shared(self.flight_2.request) + + self.begin_test_step( + "Precondition - check tested_uss has no subscription in flight 2 area" + ) + # ToDo - Add the test step details + self.end_test_step() + + st = StringBasedDateTime(datetime.utcnow()) + with OpIntentValidator( + self, + self.tested_uss_client, + self.dss, + "Validate flight 1 sharing", + self._intents_extent, + ) as validator: + _, self.flight_1_id = plan_flight( + self, + "Tested_uss plans flight 1", + self.tested_uss_client, + FlightInfo.from_scd_inject_flight_request(self.flight_1.request), + ) + + flight_1_oi_ref = validator.expect_shared( + self.flight_1.request, + ) + + control_uss_domain = "{0.scheme}://{0.netloc}/".format( + urlsplit(self.control_uss.base_url) + ) + self.begin_test_step("Validate flight2 GET interaction") + # ToDo - Add the test step details + self.end_test_step() + + self.begin_test_step("Validate flight1 Notification sent to Control_uss") + # ToDo - Add the test step details + self.end_test_step() + + delete_flight( + self, "Delete tested_uss flight", self.tested_uss_client, self.flight_1_id + ) + delete_flight( + self, "Delete control_uss flight", self.control_uss_client, self.flight_2_id + ) + + def _tested_uss_unable_to_plan_flight_near_invalid_shared_existing_flight(self): + req = self.flight_2.request + # Modifying the request with invalid data + behavior = MockUssFlightBehavior( + modify_sharing_methods=["GET", "POST"], + modify_fields={ + "reference": {"state": "Flying"}, + "details": {"priority": -1}, + }, + ) + + flight_info = FlightInfo.from_scd_inject_flight_request(req) + additional_fields = {"behavior": behavior} + + with InvalidOpIntentSharingValidator( + self, + self.control_uss_client, + self.dss, + "Validate flight 2 shared operational intent with invalid data", + self._intents_extent, + ) as validator: + _, self.flight_2_id = plan_flight( + self, + "Control_uss plans flight 2, sharing invalid operational intent data", + self.control_uss_client, + flight_info, + additional_fields, + ) + flight_2_oi_ref = validator.expect_shared_with_invalid_data(req) + + self.begin_test_step( + "Precondition - check tested_uss has no subscription in flight 2 area" + ) + # ToDo - Add the test step details + self.end_test_step() + + st = StringBasedDateTime(datetime.utcnow()) + with InvalidOpIntentSharingValidator( + self, + self.tested_uss_client, + self.dss, + "Validate flight 1 not shared by tested_uss", + self._intents_extent, + ) as validator: + _, self.flight_1_id = plan_flight_intent_expect_failed( + self, + "Tested_uss attempts to plan flight 1, expect failure", + self.tested_uss_client, + FlightInfo.from_scd_inject_flight_request(self.flight_1.request), + ) + validator.expect_not_shared() + + control_uss_domain = "{0.scheme}://{0.netloc}/".format( + urlsplit(self.control_uss.base_url) + ) + self.begin_test_step("Validate flight 2 GET interaction") + # ToDo - Add the test step details + self.end_test_step() + + self.begin_test_step("Validate flight 1 Notification not sent to Control_uss") + # ToDo - Add the test step details + self.end_test_step() + + delete_flight( + self, "Delete Control_uss flight", self.control_uss_client, self.flight_2_id + ) + def cleanup(self): self.begin_cleanup() - pass + cleanup_flights_fp_client( + self, (self.control_uss_client, self.tested_uss_client) + ), self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py new file mode 100644 index 0000000000..dc4c809512 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py @@ -0,0 +1,221 @@ +from typing import Optional, Tuple +from implicitdict import ImplicitDict +from uas_standards.astm.f3548.v21.api import ( + OperationalIntentState, + OperationalIntentReference, + GetOperationalIntentDetailsResponse, +) +from loguru import logger +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightRequest, + InjectFlightResponse, +) +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( + FlightPlannerClient, +) +from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import OpIntentValidator +from monitoring.uss_qualifier.scenarios.flight_planning.test_steps import ( + submit_flight, + expect_flight_intent_state, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType +from monitoring.monitorlib.clients.flight_planning.flight_info import FlightInfo +from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResponse, + PlanningActivityResult, + FlightPlanStatus, +) + + +def plan_flight_intent_expect_failed( + scenario: TestScenarioType, + test_step: str, + flight_planner: FlightPlannerClient, + flight_intent: FlightInfo, +) -> Tuple[PlanningActivityResponse, Optional[str]]: + """Attempt to plan a flight intent that would result in a Failed result. + + This function implements the test step described in scd_data_exchange_validation.md. + It validates requirement astm.f3548.v21.SCD00abc. + + Returns: The injection response. + """ + + return submit_flight( + scenario, + test_step, + "Plan should fail", + {(PlanningActivityResult.Failed, FlightPlanStatus.NotPlanned)}, + {}, + flight_planner, + flight_intent, + ) + + +class InvalidOpIntentSharingValidator(OpIntentValidator): + def expect_shared_with_invalid_data( + self, flight_intent: InjectFlightRequest, skip_if_not_found: bool = False + ) -> Optional[OperationalIntentReference]: + """Validate that operational intent information was shared with dss for a flight intent, but shared invalid data with USS. + + This function implements the test step described in validate_sharing_operational_intent_but_with_invalid_interuss_data. + + :param flight_intent: the flight intent that was supposed to have been shared. + :param skip_if_not_found: set to True to skip the execution of the checks if the operational intent was not found while it should have been modified. + + :returns: the shared operational intent reference. None if skipped because not found. + """ + self._begin_step() + + with self._scenario.check( + "Operational intent shared with DSS", [self._flight_planner.participant_id] + ) as check: + if self._orig_oi_ref is None: + # we expect a new op intent to have been created + if self._new_oi_ref is None: + check.record_failed( + summary="Operational intent reference not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared a new operational intent with the DSS, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[self._after_query.request.timestamp], + ) + oi_ref = self._new_oi_ref + + elif self._new_oi_ref is None: + # We expect the original op intent to have been either modified or left untouched, thus must be among + # the returned op intents. If additionally the op intent corresponds to an active flight, we fail a + # different appropriate check. Exception made if skip_if_not_found=True and op intent was deleted: step + # is skipped. + modified_oi_ref = self._find_after_oi(self._orig_oi_ref.id) + if modified_oi_ref is None: + if not skip_if_not_found: + if ( + flight_intent.operational_intent.state + == OperationalIntentState.Activated + ): + with self._scenario.check( + "Operational intent for active flight not deleted", + [self._flight_planner.participant_id], + ) as active_flight_check: + active_flight_check.record_failed( + summary="Operational intent reference for active flight not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[ + self._after_query.request.timestamp + ], + ) + else: + check.record_failed( + summary="Operational intent reference not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[self._after_query.request.timestamp], + ) + else: + self._scenario.record_note( + self._flight_planner.participant_id, + f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS, instructed to skip test step.", + ) + self._scenario.end_test_step() + return None + oi_ref = modified_oi_ref + + else: + # we expect the original op intent to have been replaced with a new one, thus old one must NOT be among the returned op intents + if self._find_after_oi(self._orig_oi_ref.id) is not None: + check.record_failed( + summary="Operational intent reference found duplicated in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by replacing it, but it ended up duplicating the operational intent in the DSS", + query_timestamps=[self._after_query.request.timestamp], + ) + oi_ref = self._new_oi_ref + + goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( + oi_ref + ) + self._scenario.record_query(oi_full_query) + with self._scenario.check( + "Operational intent details retrievable", + [self._flight_planner.participant_id], + ) as check: + if oi_full_query.status_code != 200: + check.record_failed( + summary="Operational intent details could not be retrieved from USS", + severity=Severity.High, + details=f"Received status code {oi_full_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {oi_ref.id}", + query_timestamps=[oi_full_query.request.timestamp], + ) + + # if schema validation errors or standard req validation + with self._scenario.check( + "Invalid data in Operational intent details shared by Mock USS for negative test", + [self._flight_planner.participant_id], + ) as check: + + validation_errors = [] + # schema_validation_errors = schema_validation.validate( + # schema_validation.F3548_21.OpenAPIPath, + # schema_validation.F3548_21.GetOperationalIntentDetailsResponse, + # oi_full_query.response.json, + # ) + schema_validation_errors = None + logger.debug(f"Schema validation errors {schema_validation_errors}") + if schema_validation_errors: + details = ( + "The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" + + "\n".join( + f"At {e.json_path} in the response: {e.message}" + for e in schema_validation_errors + ) + ) + validation_errors.append(details) + else: + oi_full = None + try: + goidr = ImplicitDict.parse( + goidr_json, GetOperationalIntentDetailsResponse + ) + oi_full = goidr.operational_intent + + if ( + oi_full.reference.state == OperationalIntentState.Accepted + or oi_full.reference.state == OperationalIntentState.Activated + ) and oi_full.details.get("off_nominal_volumes", None): + details = f"Operational intent {oi_full.reference.id} had {len(oi_full.details.off_nominal_volumes)} off-nominal volumes in wrong state - {oi_full.reference.state}" + validation_errors.append(details) + + def volume_vertices(v4): + if "outline_circle" in v4.volume: + return 1 + if "outline_polygon" in v4.volume: + return len(v4.volume.outline_polygon.vertices) + + all_volumes = oi_full.details.get( + "volumes", [] + ) + oi_full.details.get("off_nominal_volumes", []) + n_vertices = sum(volume_vertices(v) for v in all_volumes) + + if n_vertices > 10000: + details = ( + f"Operational intent {oi_full.reference.id} had too many total vertices - {n_vertices}", + ) + validation_errors.append(details) + except (KeyError, ValueError) as e: + logger.debug( + f"Validation error in GetOperationalIntentDetailsResponse. {e}" + ) + validation_errors.append(e) + + if not validation_errors: + check.record_failed( + summary="This negative test case requires invalid data shared with other USS in Operational intent details ", + severity=Severity.High, + details=f"Data shared by Mock USS with other USSes had no invalid data. This test case required invalid data for testing.", + query_timestamps=[oi_full_query.request.timestamp], + ) + + self._scenario.end_test_step() + return oi_ref diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index 8370d1b830..1021a20c1f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import List, Optional +from typing import List, Optional, Union from monitoring.monitorlib import schema_validation, fetch from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient @@ -46,7 +46,7 @@ class OpIntentValidator(object): def __init__( self, scenario: TestScenarioType, - flight_planner: FlightPlanner | FlightPlannerClient, + flight_planner: Union[FlightPlanner, FlightPlannerClient], dss: DSSInstance, test_step: str, extent: Volume4D, @@ -61,7 +61,7 @@ def __init__( :param orig_oi_ref: if this is validating a previously existing operational intent (e.g. modification), pass the original reference. """ self._scenario: TestScenarioType = scenario - self._flight_planner: FlightPlanner | FlightPlannerClient = flight_planner + self._flight_planner: Union[FlightPlanner, FlightPlannerClient] = flight_planner self._dss: DSSInstance = dss self._test_step: str = test_step self._extent: Volume4D = extent diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 459b434034..33e7670000 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -1,8 +1,8 @@ import inspect from typing import Optional, Tuple, Iterable, Set, Dict, Union -from monitoring.monitorlib.geotemporal import Volume4DCollection from monitoring.monitorlib.clients.flight_planning.client import PlanningActivityError from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResponse, PlanningActivityResult, FlightPlanStatus, ) @@ -53,6 +53,7 @@ def plan_flight_intent( flight_intent: InjectFlightRequest, ) -> Tuple[InjectFlightResponse, Optional[str]]: """Plan a flight intent that should result in success. + Note: This method will be deprecated in favor of plan_flight This function implements the test step described in plan_flight_intent.md. @@ -202,6 +203,8 @@ def submit_flight_intent( flight_id: Optional[str] = None, ) -> Tuple[InjectFlightResponse, Optional[str]]: """Submit a flight intent with an expected result. + Note: This method will be deprecated in favor of submit_flight + A check fail is considered by default of high severity and as such will raise an ScenarioCannotContinueError. The severity of each failed check may be overridden if needed. @@ -270,6 +273,8 @@ def delete_flight_intent( flight_id: str, ) -> DeleteFlightResponse: """Delete an existing flight intent that should result in success. + Note: This method will be deprecated in favor of delete_flight + A check fail is considered of high severity and as such will raise an ScenarioCannotContinueError. This function implements the test step described in `delete_flight_intent.md`. @@ -314,6 +319,7 @@ def cleanup_flights( scenario: TestScenarioType, flight_planners: Iterable[FlightPlanner] ) -> None: """Remove flights during a cleanup test step. + Note: This method will be deprecated in favor of cleanup_flights_fp_client This function assumes: * `scenario` is currently cleaning up (cleanup has started) @@ -359,7 +365,7 @@ def plan_flight( flight_planner: FlightPlannerClient, flight_info: FlightInfo, additional_fields: Optional[dict] = None, -) -> Tuple[InjectFlightResponse, Optional[str]]: +) -> Tuple[PlanningActivityResponse, Optional[str]]: """Plan a flight intent that should result in success. This function implements the test step described in @@ -373,8 +379,8 @@ def plan_flight( scenario=scenario, test_step=test_step, success_check="Successful planning", - expected_results={InjectFlightResponseResult.Planned}, - failed_checks={InjectFlightResponseResult.Failed: "Failure"}, + expected_results={(PlanningActivityResult.Completed, FlightPlanStatus.Planned)}, + failed_checks={PlanningActivityResult.Failed: "Failure"}, flight_planner=flight_planner, flight_info=flight_info, additional_fields=additional_fields, @@ -385,13 +391,13 @@ def submit_flight( scenario: TestScenarioType, test_step: str, success_check: str, - expected_results: Set[InjectFlightResponseResult], - failed_checks: Dict[InjectFlightResponseResult, Union[str, Tuple[str, Severity]]], + expected_results: Set[Tuple[PlanningActivityResult, FlightPlanStatus]], + failed_checks: Dict[PlanningActivityResult, Union[str, Tuple[str, Severity]]], flight_planner: FlightPlannerClient, flight_info: FlightInfo, flight_id: Optional[str] = None, additional_fields: Optional[dict] = None, -) -> Tuple[InjectFlightResponse, Optional[str]]: +) -> Tuple[PlanningActivityResponse, Optional[str]]: """Submit a flight intent with an expected result. A check fail is considered by default of high severity and as such will raise an ScenarioCannotContinueError. The severity of each failed check may be overridden if needed. @@ -431,7 +437,7 @@ def submit_flight( with scenario.check( check_name, [flight_planner.participant_id] ) as specific_failed_check: - if resp.result == unexpected_result: + if resp.activity_result == unexpected_result: specific_failed_check.record_failed( summary=f"Flight unexpectedly {resp.result}", severity=check_severity, @@ -439,7 +445,7 @@ def submit_flight( query_timestamps=[query.request.timestamp], ) - if resp.result in expected_results: + if (resp.activity_result, resp.flight_plan_status) in expected_results: scenario.end_test_step() return resp, flight_id else: @@ -460,7 +466,7 @@ def request_flight( flight_info: FlightInfo, flight_id: Optional[str], additional_fields: Optional[dict] = None, -) -> Tuple[InjectFlightResponse, Query, str]: +) -> Tuple[PlanningActivityResponse, Query, str]: """ This method is needed till we are able to have checks with PlanningActivityResult. Uses FlightPlannerClient to plan the flight @@ -488,39 +494,12 @@ def request_flight( except PlanningActivityError as e: raise QueryError(str(e), e.queries) - if resp.activity_result == PlanningActivityResult.Failed: - result = InjectFlightResponseResult.Failed - elif resp.activity_result == PlanningActivityResult.NotSupported: - result = InjectFlightResponseResult.NotSupported - elif resp.activity_result == PlanningActivityResult.Rejected: - result = InjectFlightResponseResult.Rejected - elif resp.activity_result == PlanningActivityResult.Completed: - if resp.flight_plan_status == FlightPlanStatus.Planned: - result = InjectFlightResponseResult.Planned - elif resp.flight_plan_status == FlightPlanStatus.OkToFly: - result = InjectFlightResponseResult.ReadyToFly - elif resp.flight_plan_status == FlightPlanStatus.OffNominal: - result = InjectFlightResponseResult.ReadyToFly - else: - raise NotImplementedError( - f"Unable to handle '{resp.flight_plan_status}' FlightPlanStatus with {resp.activity_result} PlanningActivityResult" - ) - else: - raise NotImplementedError( - f"Unable to handle '{resp.activity_result}' PlanningActivityResult" - ) - - response = InjectFlightResponse( - result=result, - operational_intent_id="", - ) - - return response, resp.queries[0], flight_id + return resp, resp.queries[0], flight_id def cleanup_flight( flight_planner: FlightPlannerClient, flight_id: str -) -> Tuple[DeleteFlightResponse, Query]: +) -> Tuple[PlanningActivityResponse, Query]: """ This method is required till we are able to have checks with PlanningActivityResult Args: @@ -535,20 +514,11 @@ def cleanup_flight( except PlanningActivityError as e: raise QueryError(str(e), e.queries) - if ( - resp.activity_result == PlanningActivityResult.Completed - and resp.flight_plan_status == FlightPlanStatus.Closed - ): - flight_planner.created_flight_ids.discard(str(flight_id)) - return ( - DeleteFlightResponse(result=DeleteFlightResponseResult.Closed), - resp.queries[0], - ) - else: - return ( - DeleteFlightResponse(result=DeleteFlightResponseResult.Failed), - resp.queries[0], - ) + flight_planner.created_flight_ids.discard(str(flight_id)) + return ( + resp, + resp.queries[0], + ) def delete_flight( @@ -556,7 +526,7 @@ def delete_flight( test_step: str, flight_planner: FlightPlannerClient, flight_id: str, -) -> DeleteFlightResponse: +) -> PlanningActivityResponse: """Delete an existing flight intent that should result in success. A check fail is considered of high severity and as such will raise an ScenarioCannotContinueError. @@ -582,14 +552,17 @@ def delete_flight( scenario.record_query(query) notes_suffix = f': "{resp.notes}"' if "notes" in resp and resp.notes else "" - if resp.result == DeleteFlightResponseResult.Closed: + if ( + resp.activity_result == PlanningActivityResult.Completed + and resp.flight_plan_status == FlightPlanStatus.Closed + ): scenario.end_test_step() return resp else: check.record_failed( - summary=f"Flight deletion attempt unexpectedly {resp.result}", + summary=f"Flight deletion attempt unexpectedly {(resp.activity_result,resp.flight_plan_status)}", severity=Severity.High, - details=f"{flight_planner.participant_id} indicated {resp.result} rather than the expected {DeleteFlightResponseResult.Closed}{notes_suffix}", + details=f"{flight_planner.participant_id} indicated {(resp.activity_result,resp.flight_plan_status)} rather than the expected {PlanningActivityResult.Completed,FlightPlanStatus.Closed}{notes_suffix}", query_timestamps=[query.request.timestamp], ) @@ -602,6 +575,7 @@ def cleanup_flights_fp_client( scenario: TestScenarioType, flight_planners: Iterable[FlightPlannerClient] ) -> None: """Remove flights during a cleanup test step. + Note: This method should be renamed to cleanup_flights once deprecated cleanup_flights method is removed This function assumes: * `scenario` is currently cleaning up (cleanup has started) @@ -628,7 +602,10 @@ def cleanup_flights_fp_client( ) continue - if resp.result == DeleteFlightResponseResult.Closed: + if ( + resp.activity_result == PlanningActivityResult.Completed + and resp.flight_plan_status == FlightPlanStatus.Closed + ): removed.append(flight_id) else: check.record_failed(