From 2ca7dee5c4f881fe08c6d4572a40c755b1d98ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Mon, 14 Oct 2024 14:02:34 +0200 Subject: [PATCH] [uss_qualifier/scenarios/data_exchange_validation] Fix wrong check failure for lack of notification (#800) --- .../get_op_data_validation.py | 15 ++++-- .../expected_interactions_test_steps.py | 52 ++++++++++++++----- ...date_no_notification_operational_intent.md | 3 ++ 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index af7b18a7ee..90d1e4d2b6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict +from typing import Optional, Dict, Set from monitoring.monitorlib.clients.flight_planning.flight_info import ( AirspaceUsageState, @@ -57,7 +57,7 @@ submit_flight, ) from monitoring.uss_qualifier.suites.suite import ExecutionContext -from uas_standards.astm.f3548.v21.api import OperationID +from uas_standards.astm.f3548.v21.api import OperationID, EntityID from uas_standards.astm.f3548.v21.constants import Scope @@ -65,6 +65,8 @@ class GetOpResponseDataValidationByUSS(TestScenario): flight_1: FlightInfoTemplate flight_2: FlightInfoTemplate + op_intent_ids: Set[EntityID] + tested_uss_client: FlightPlannerClient mock_uss: MockUSSClient mock_uss_client: FlightPlannerClient @@ -128,6 +130,7 @@ def __init__( setattr(self, efi.intent_id, templates[efi.intent_id]) def run(self, context: ExecutionContext): + self.op_intent_ids = set() times = { TimeDuringTest.StartOfTestRun: Time(context.start_time), TimeDuringTest.StartOfScenario: Time(arrow.utcnow().datetime), @@ -168,6 +171,7 @@ def _plan_successfully_test_case(self, times: Dict[TimeDuringTest, Time]): ) flight_2_oi_ref = validator.expect_shared(flight_2) + self.op_intent_ids.add(flight_2_oi_ref.id) self.end_test_step() times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) @@ -186,9 +190,8 @@ def _plan_successfully_test_case(self, times: Dict[TimeDuringTest, Time]): self.tested_uss_client, flight_1, ) - validator.expect_shared( - flight_1, - ) + flight_1_oi_ref = validator.expect_shared(flight_1) + self.op_intent_ids.add(flight_1_oi_ref.id) self.end_test_step() self.begin_test_step( @@ -286,6 +289,7 @@ def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]): validation_failure_type=OpIntentValidationFailureType.DataFormat, invalid_fields=[modify_field1, modify_field2], ) + self.op_intent_ids.add(flight_2_oi_ref.id) self.end_test_step() times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime) @@ -357,6 +361,7 @@ def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]): self, self.mock_uss, flight_1_planning_time, + self.op_intent_ids, self.tested_uss_client.participant_id, ) self.end_test_step() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py index dd1024a3a8..6868ff7e1e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py @@ -2,12 +2,17 @@ from datetime import datetime, timedelta import re -from typing import Callable, Dict, List, Tuple, Optional +from typing import Callable, Dict, List, Tuple, Optional, Set import arrow -from implicitdict import StringBasedDateTime +from implicitdict import StringBasedDateTime, ImplicitDict from uas_standards.astm.f3548.v21 import api -from uas_standards.astm.f3548.v21.api import OperationID, EntityID +from uas_standards.astm.f3548.v21.api import ( + OperationID, + EntityID, + PutOperationalIntentDetailsParameters, + OperationalIntentReference, +) from monitoring.monitorlib.clients.mock_uss.interactions import Interaction from monitoring.monitorlib.clients.mock_uss.interactions import QueryDirection @@ -58,32 +63,55 @@ def expect_no_interuss_post_interactions( scenario: TestScenarioType, mock_uss: MockUSSClient, st: StringBasedDateTime, + shared_op_intent_ids: Set[EntityID], participant_id: str, ): - """This step checks no notification is sent to any USS within the required time window (as no DSS entity was created). + """This step checks no notification about an unexpected operational intent is sent to any USS within the required time window (as no DSS entity was created). Args: st: the earliest time a notification may have been sent + shared_op_intent_ids: the set of IDs of previously shared operational intents for which it is expected that notifications are present regardless of their timings participant_id: id of the participant responsible to send the notification """ sleep( max_wait_time, "we have to wait the longest it may take a USS to send a notification before we can establish that they didn't send a notification", ) - found, query = mock_uss_interactions( + interactions, query = mock_uss_interactions( scenario=scenario, mock_uss=mock_uss, op_id=OperationID.NotifyOperationalIntentDetailsChanged, direction=QueryDirection.Incoming, since=st, ) - with scenario.check("Expect Notification not sent", [participant_id]) as check: - if found: - check.record_failed( - summary=f"Notification was wrongly sent for an entity not created.", - details=f"Notification was wrongly sent for an entity not created.", - query_timestamps=[query.request.timestamp], - ) + + for interaction in interactions: + with scenario.check( + "Mock USS interaction can be parsed", [mock_uss.participant_id] + ) as check: + try: + req = PutOperationalIntentDetailsParameters( + ImplicitDict.parse( + interaction.query.request.json, + PutOperationalIntentDetailsParameters, + ) + ) + except (ValueError, TypeError, KeyError) as e: + check.record_failed( + summary=f"Failed to parse request of a 'NotifyOperationalIntentDetailsChanged' interaction with mock_uss as a PutOperationalIntentDetailsParameters", + details=f"{str(e)}\nRequest: {interaction.query.request.json}\n\nStack trace:\n{e.stacktrace}", + query_timestamps=[query.request.timestamp], + ) + continue # low priority failure: continue checking interactions if one cannot be parsed + + with scenario.check("Expect Notification not sent", [participant_id]) as check: + op_intent_id = EntityID(req.operational_intent_id) + if op_intent_id not in shared_op_intent_ids: + check.record_failed( + summary=f"Observed unexpected notification for operational intent ID {req.operational_intent_id}.", + details=f"Notification for operational intent ID {req.operational_intent_id} triggered by subscriptions {', '.join([sub.subscription_id for sub in req.subscriptions])} with timestamp {interaction.query.request.timestamp}.", + query_timestamps=[query.request.timestamp], + ) def mock_uss_interactions( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md index 3f760484bb..28d5b9c3ba 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md @@ -5,6 +5,9 @@ This step verifies when a flight is not created, it is also not notified by chec ## 🛑 Mock USS interactions logs retrievable check **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +## ℹī¸ Mock USS interaction can be parsed check +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + ## 🛑 Expect Notification not sent check **[interuss.f3548.notification_requirements.NoDssEntityNoNotification](../../../../../requirements/interuss/f3548/notification_requirements.md)**