[uss_qualifier/scenarios/data_exchange_validation] Fix wrong check fa…

…ilure for lack of notification (interuss#800)

monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py

@@ -1,4 +1,4 @@
-from typing import Optional, Dict
+from typing import Optional, Dict, Set
 
 from monitoring.monitorlib.clients.flight_planning.flight_info import (
     AirspaceUsageState,
@@ -57,14 +57,16 @@
     submit_flight,
 )
 from monitoring.uss_qualifier.suites.suite import ExecutionContext
-from uas_standards.astm.f3548.v21.api import OperationID
+from uas_standards.astm.f3548.v21.api import OperationID, EntityID
 from uas_standards.astm.f3548.v21.constants import Scope
 
 
 class GetOpResponseDataValidationByUSS(TestScenario):
     flight_1: FlightInfoTemplate
     flight_2: FlightInfoTemplate
 
+    op_intent_ids: Set[EntityID]
+
     tested_uss_client: FlightPlannerClient
     mock_uss: MockUSSClient
     mock_uss_client: FlightPlannerClient
@@ -128,6 +130,7 @@ def __init__(
             setattr(self, efi.intent_id, templates[efi.intent_id])
 
     def run(self, context: ExecutionContext):
+        self.op_intent_ids = set()
         times = {
             TimeDuringTest.StartOfTestRun: Time(context.start_time),
             TimeDuringTest.StartOfScenario: Time(arrow.utcnow().datetime),
@@ -168,6 +171,7 @@ def _plan_successfully_test_case(self, times: Dict[TimeDuringTest, Time]):
             )
 
             flight_2_oi_ref = validator.expect_shared(flight_2)
+            self.op_intent_ids.add(flight_2_oi_ref.id)
         self.end_test_step()
 
         times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime)
@@ -186,9 +190,8 @@ def _plan_successfully_test_case(self, times: Dict[TimeDuringTest, Time]):
                 self.tested_uss_client,
                 flight_1,
             )
-            validator.expect_shared(
-                flight_1,
-            )
+            flight_1_oi_ref = validator.expect_shared(flight_1)
+            self.op_intent_ids.add(flight_1_oi_ref.id)
         self.end_test_step()
 
         self.begin_test_step(
@@ -286,6 +289,7 @@ def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]):
                 validation_failure_type=OpIntentValidationFailureType.DataFormat,
                 invalid_fields=[modify_field1, modify_field2],
             )
+            self.op_intent_ids.add(flight_2_oi_ref.id)
         self.end_test_step()
 
         times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime)
@@ -357,6 +361,7 @@ def _plan_unsuccessfully_test_case(self, times: Dict[TimeDuringTest, Time]):
             self,
             self.mock_uss,
             flight_1_planning_time,
+            self.op_intent_ids,
             self.tested_uss_client.participant_id,
         )
         self.end_test_step()

monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py

@@ -2,12 +2,17 @@
 
 from datetime import datetime, timedelta
 import re
-from typing import Callable, Dict, List, Tuple, Optional
+from typing import Callable, Dict, List, Tuple, Optional, Set
 
 import arrow
-from implicitdict import StringBasedDateTime
+from implicitdict import StringBasedDateTime, ImplicitDict
 from uas_standards.astm.f3548.v21 import api
-from uas_standards.astm.f3548.v21.api import OperationID, EntityID
+from uas_standards.astm.f3548.v21.api import (
+    OperationID,
+    EntityID,
+    PutOperationalIntentDetailsParameters,
+    OperationalIntentReference,
+)
 
 from monitoring.monitorlib.clients.mock_uss.interactions import Interaction
 from monitoring.monitorlib.clients.mock_uss.interactions import QueryDirection
@@ -58,32 +63,55 @@ def expect_no_interuss_post_interactions(
     scenario: TestScenarioType,
     mock_uss: MockUSSClient,
     st: StringBasedDateTime,
+    shared_op_intent_ids: Set[EntityID],
     participant_id: str,
 ):
-    """This step checks no notification is sent to any USS within the required time window (as no DSS entity was created).
+    """This step checks no notification about an unexpected operational intent is sent to any USS within the required time window (as no DSS entity was created).
 
     Args:
         st: the earliest time a notification may have been sent
+        shared_op_intent_ids: the set of IDs of previously shared operational intents for which it is expected that notifications are present regardless of their timings
         participant_id: id of the participant responsible to send the notification
     """
     sleep(
         max_wait_time,
         "we have to wait the longest it may take a USS to send a notification before we can establish that they didn't send a notification",
     )
-    found, query = mock_uss_interactions(
+    interactions, query = mock_uss_interactions(
         scenario=scenario,
         mock_uss=mock_uss,
         op_id=OperationID.NotifyOperationalIntentDetailsChanged,
         direction=QueryDirection.Incoming,
         since=st,
     )
-    with scenario.check("Expect Notification not sent", [participant_id]) as check:
-        if found:
-            check.record_failed(
-                summary=f"Notification was wrongly sent for an entity not created.",
-                details=f"Notification was wrongly sent for an entity not created.",
-                query_timestamps=[query.request.timestamp],
-            )
+
+    for interaction in interactions:
+        with scenario.check(
+            "Mock USS interaction can be parsed", [mock_uss.participant_id]
+        ) as check:
+            try:
+                req = PutOperationalIntentDetailsParameters(
+                    ImplicitDict.parse(
+                        interaction.query.request.json,
+                        PutOperationalIntentDetailsParameters,
+                    )
+                )
+            except (ValueError, TypeError, KeyError) as e:
+                check.record_failed(
+                    summary=f"Failed to parse request of a 'NotifyOperationalIntentDetailsChanged' interaction with mock_uss as a PutOperationalIntentDetailsParameters",
+                    details=f"{str(e)}\nRequest: {interaction.query.request.json}\n\nStack trace:\n{e.stacktrace}",
+                    query_timestamps=[query.request.timestamp],
+                )
+                continue  # low priority failure: continue checking interactions if one cannot be parsed
+
+        with scenario.check("Expect Notification not sent", [participant_id]) as check:
+            op_intent_id = EntityID(req.operational_intent_id)
+            if op_intent_id not in shared_op_intent_ids:
+                check.record_failed(
+                    summary=f"Observed unexpected notification for operational intent ID {req.operational_intent_id}.",
+                    details=f"Notification for operational intent ID {req.operational_intent_id} triggered by subscriptions {', '.join([sub.subscription_id for sub in req.subscriptions])} with timestamp {interaction.query.request.timestamp}.",
+                    query_timestamps=[query.request.timestamp],
+                )
 
 
 def mock_uss_interactions(

monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md

@@ -5,6 +5,9 @@ This step verifies when a flight is not created, it is also not notified by chec
 ## 🛑 Mock USS interactions logs retrievable check
 **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**.
 
+## ℹ️ Mock USS interaction can be parsed check
+**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**.
+
 ## 🛑 Expect Notification not sent check
 
 **[interuss.f3548.notification_requirements.NoDssEntityNoNotification](../../../../../requirements/interuss/f3548/notification_requirements.md)**