diff --git a/monitoring/monitorlib/mutate/scd.py b/monitoring/monitorlib/mutate/scd.py index 3543fe2c58..818f27470c 100644 --- a/monitoring/monitorlib/mutate/scd.py +++ b/monitoring/monitorlib/mutate/scd.py @@ -9,6 +9,7 @@ OperationID, Subscription, PutSubscriptionParameters, + OperationalIntentReference, ) from yaml.representer import Representer @@ -51,6 +52,22 @@ def subscription(self) -> Optional[Subscription]: except ValueError: return None + @property + def operational_intent_references(self) -> List[OperationalIntentReference]: + if self.json_result is None: + return [] + try: + if "operational_intent_references" not in self.json_result: + return [] + oirs_json = self.json_result["operational_intent_references"] + if not isinstance(oirs_json, list): + return [] + return [ + ImplicitDict.parse(oir, OperationalIntentReference) for oir in oirs_json + ] + except ValueError: + return [] + yaml.add_representer(MutatedSubscription, Representer.represent_dict) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read_correct.md similarity index 70% rename from monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md rename to monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read_correct.md index bc0384923a..9220a859c6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read_correct.md @@ -2,9 +2,9 @@ This test step fragment validates that subscriptions can be read. -## 🛑 Get Subscription by ID check +## [Read query succeeds](./read_query.md) -If a subscription cannot be queried using its ID, the DSS is failing to meet **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. +Check query succeeds. ## 🛑 Get subscription response format conforms to spec check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read_query.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read_query.md new file mode 100644 index 0000000000..ce8e39652b --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/read_query.md @@ -0,0 +1,7 @@ +# Read subscription query test step fragment + +This test step fragment validates that a query to read a subscription succeeds. + +## 🛑 Get Subscription by ID check + +If a subscription cannot be queried using its ID, the DSS is failing to meet **[astm.f3548.v21.DSS0005,5](../../../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.md index eab95740e3..9f410123a6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.md @@ -79,4 +79,34 @@ If the DSS omits any of the implicit subscriptions belonging to an OIR previousl any of the DSSes at which an earlier OIR was created, or the DSS at which the current OIR has been created, are in violation of **[astm.f3548.v21.DSS0210,A2-7-2,4b](../../../../requirements/astm/f3548/v21.md)**. +## Subscription creation returns relevant OIRs test case + +This test case checks that, when a newly created subscription intersects with an existing OIR and that the subscription is intended for operational intent references, +the DSS includes the relevant OIRs in the response to the creation. + +### Create a subscription at every DSS in sequence test step + +This test step will create a new subscription at every DSS, in sequence, each time verifying that the DSS +returns any OIRs that intersect with the newly created subscription. + +Note that this step is run once for each involved DSS (that is, once for the primary DSS and once for every secondary DSS) + +#### [Create subscription on a DSS instance](./fragments/sub/crud/create_query.md) + +Check that the subscription creation succeeds. + +#### 🛑 DSS response contains the expected OIRs check + +The response from a DSS to a valid subscription creation request is expected to contain any relevant OIRs for the subscription's extents if the subscription had the `notify_for_op_intents` flag set to `true`. + +If the DSS omits the intersecting OIR, it fails to comply with **[astm.f3548.v21.DSS0210,A2-7-2,4a](../../../../requirements/astm/f3548/v21.md)**. + +#### [Get subscription query from all other DSS instances succeeds](./fragments/sub/crud/read_query.md) + +#### 🛑 Subscription may be retrieved from all other DSS instances check + +The subscription created on a DSS instance must be retrievable from all other DSS instances. + +If the subscription does not exist on one of the other DSS instances, one of the instances fails to comply with **[astm.f3548.v21.DSS0210,A2-7-2,4a](../../../../requirements/astm/f3548/v21.md)**. + ## [Cleanup](./clean_workspace.md) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.py index 67f8ddcebe..3047e0b288 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/subscription_interactions.py @@ -12,10 +12,9 @@ ) from uas_standards.astm.f3548.v21.constants import Scope -from monitoring.monitorlib.delay import sleep +from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import QueryError, Query from monitoring.monitorlib.geotemporal import Volume4D -from monitoring.monitorlib.mutate.scd import MutatedSubscription from monitoring.prober.infrastructure import register_resource_type from monitoring.uss_qualifier.resources.astm.f3548.v21 import PlanningAreaResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import ( @@ -31,7 +30,6 @@ from monitoring.uss_qualifier.scenarios.astm.utm.dss import test_step_fragments from monitoring.uss_qualifier.scenarios.scenario import ( TestScenario, - PendingCheck, ) from monitoring.uss_qualifier.suites.suite import ExecutionContext @@ -44,14 +42,19 @@ class SubscriptionInteractions(TestScenario): A scenario that tests interactions between subscriptions and entities across a DSS cluster. """ - SUB_TYPES = [ - register_resource_type(386, "First Subscription"), - register_resource_type(387, "Second Subscription"), + BG_SUB_TYPES = [ + register_resource_type(386, "Background subscription 1"), + register_resource_type(387, "Background subscription 2"), ] - OIR_TYPE = register_resource_type(388, "Operational Intent References") + PER_DSS_OIR_TYPE = register_resource_type( + 388, "Multiple Operational Intent References" + ) + PER_DSS_SUB_TYPE = register_resource_type(389, "Multiple Subscriptions") + + _background_sub_ids: List[SubscriptionID] - _sub_ids: List[SubscriptionID] _oir_ids: List[EntityID] + _sub_ids: List[SubscriptionID] _current_subs: Dict[SubscriptionID, Subscription] _current_oirs: Dict[EntityID, OperationalIntentReference] @@ -92,17 +95,23 @@ def __init__( dss.get_instance(scopes) for dss in other_instances.dss_instances ] - # Prepare the two subscription ids: - self._sub_ids = [ - id_generator.id_factory.make_id(sub_type) for sub_type in self.SUB_TYPES + # Prepare the two background subscription ids: + self._background_sub_ids = [ + id_generator.id_factory.make_id(sub_type) for sub_type in self.BG_SUB_TYPES ] # Prepare one OIR id for each DSS we will interact with (one for the main and one for each secondary) - base_oir_id = id_generator.id_factory.make_id(self.OIR_TYPE) + base_oir_id = id_generator.id_factory.make_id(self.PER_DSS_OIR_TYPE) self._oir_ids = [ f"{base_oir_id[:-3]}{i:03d}" for i in range(len(self._secondary_instances) + 1) ] + # Prepare one subscription id for each DSS we will interact with (one for the main and one for each secondary) + base_sub_id = id_generator.id_factory.make_id(self.PER_DSS_SUB_TYPE) + self._sub_ids = [ + f"{base_sub_id[:-3]}{i:03d}" + for i in range(len(self._secondary_instances) + 1) + ] self._manager = utm_client_identity.subject() @@ -115,6 +124,10 @@ def run(self, context: ExecutionContext): self._steps_create_oirs_at_each_dss() self.end_test_case() + self.begin_test_case("Subscription creation returns relevant OIRs") + self._steps_create_subs_at_each_dss() + self.end_test_case() + self.end_test_scenario() def _step_create_background_subs(self): @@ -125,7 +138,7 @@ def _step_create_background_subs(self): self.begin_test_step("Create first background subscription") sub_now_params = self._planning_area.get_new_subscription_params( - subscription_id=self._sub_ids[0], + subscription_id=self._background_sub_ids[0], start_time=self._sub_1_start, duration=self._sub_1_end - self._sub_1_start, # This is a planning area without constraint processing @@ -133,7 +146,7 @@ def _step_create_background_subs(self): notify_for_constraints=False, ) - sub_now = self._create_sub_with_params(sub_now_params) + sub_now, _, _ = self._create_sub_with_params(sub_now_params) self._current_subs[sub_now_params.sub_id] = sub_now self.end_test_step() @@ -141,7 +154,7 @@ def _step_create_background_subs(self): self.begin_test_step("Create second background subscription") sub_later_params = self._planning_area.get_new_subscription_params( - subscription_id=self._sub_ids[1], + subscription_id=self._background_sub_ids[1], start_time=self._sub_2_start, duration=self._sub_2_end - self._sub_2_start, # This is a planning area without constraint processing @@ -149,7 +162,7 @@ def _step_create_background_subs(self): notify_for_constraints=False, ) - sub_later = self._create_sub_with_params(sub_later_params) + sub_later, _, _ = self._create_sub_with_params(sub_later_params) self._current_subs[sub_later_params.sub_id] = sub_later self.end_test_step() @@ -185,10 +198,10 @@ def _steps_create_oirs_at_each_dss(self): "DSS response contains the expected background subscription", dss.participant_id, ) as check: - if self._sub_ids[0] not in notification_ids: + if self._background_sub_ids[0] not in notification_ids: check.record_failed( summary="DSS did not return the intersecting background subscription", - details=f"Expected subscription {self._sub_ids[0]} (first background subscription) in the" + details=f"Expected subscription {self._background_sub_ids[0]} (first background subscription) in the" f" list of subscriptions to notify, but got {notification_ids}", query_timestamps=[q.request.timestamp], ) @@ -197,10 +210,10 @@ def _steps_create_oirs_at_each_dss(self): "DSS does not return non-intersecting background subscription", dss.participant_id, ) as check: - if self._sub_ids[1] in notification_ids: + if self._background_sub_ids[1] in notification_ids: check.record_failed( summary="DSS returned the non-intersecting background subscription", - details=f"Expected subscription {self._sub_ids[1]} (second background subscription) to not be in the" + details=f"Expected subscription {self._background_sub_ids[1]} (second background subscription) to not be in the" f" list of subscriptions to notify, but got {notification_ids}", query_timestamps=[q.request.timestamp], ) @@ -223,6 +236,71 @@ def _steps_create_oirs_at_each_dss(self): self._current_oirs[oir_id] = oir self.end_test_step() + def _steps_create_subs_at_each_dss(self): + """Creates a subscription at each DSS instance""" + + # The new subscriptions use the same parameters as the first background subscription + common_params = self._planning_area.get_new_subscription_params( + subscription_id="", + start_time=self._sub_1_start, + duration=self._sub_1_end - self._sub_1_start, + notify_for_op_intents=True, + notify_for_constraints=False, + ) + + # All previously created OIRs are relevant to each subscription + expected_oir_ids = set(self._oir_ids) + + for i, dss in enumerate([self._dss] + self._secondary_instances): + self.begin_test_step("Create a subscription at every DSS in sequence") + + sub_id = self._sub_ids[i] + common_params.sub_id = sub_id + sub, oirs, r = self._create_sub_with_params(common_params) + self._current_subs[sub_id] = sub + + returned_oir_ids = set(oir.id for oir in oirs) + + with self.check( + "DSS response contains the expected OIRs", + dss.participant_id, + ) as check: + if not expected_oir_ids.issubset(returned_oir_ids): + missing_oirs = expected_oir_ids - returned_oir_ids + check.record_failed( + summary="DSS did not return the expected OIRs", + details=f"Expected OIRs {expected_oir_ids} in the list of OIRs to notify, but got {returned_oir_ids}. " + f"Missing: {missing_oirs}", + query_timestamps=[r.request.timestamp], + ) + + for other_dss in {self._dss, *self._secondary_instances} - {dss}: + other_dss_sub = other_dss.get_subscription(sub_id) + with self.check( + "Get Subscription by ID", + other_dss.participant_id, + ) as check: + if not other_dss_sub.success: + check.record_failed( + summary="Get subscription query failed", + details=f"Failed to retrieved a subscription from DSS with code {other_dss_sub.status_code}: {other_dss_sub.error_message}", + query_timestamps=[other_dss_sub.request.timestamp], + ) + + with self.check( + "Subscription may be retrieved from all other DSS instances", + [dss.participant_id, other_dss.participant_id], + ) as check: + # status may have been 404 + if other_dss_sub.status_code != 200: + check.record_failed( + summary="Subscription created on a DSS instance was not found on another instance", + details=f"Subscription {sub_id} created on DSS instance {dss.participant_id} was not found on DSS instance {other_dss.participant_id} (error message: {other_dss_sub.error_message}).", + query_timestamps=[other_dss_sub.request.timestamp], + ) + + self.end_test_step() + def _put_op_intent( self, dss: DSSInstance, @@ -253,7 +331,9 @@ def _put_op_intent( return oir, subs, q - def _create_sub_with_params(self, params: SubscriptionParams) -> Subscription: + def _create_sub_with_params( + self, params: SubscriptionParams + ) -> Tuple[Subscription, List[OperationalIntentReference], fetch.Query]: """Create a subscription with the given parameters via the primary DSS instance""" with self.check("Create subscription query succeeds") as check: r = self._dss.upsert_subscription(**params) @@ -263,7 +343,7 @@ def _create_sub_with_params(self, params: SubscriptionParams) -> Subscription: details=f"Failed to create a subscription on primary DSS with code {r.status_code}: {r.error_message}", query_timestamps=[r.request.timestamp], ) - return r.subscription + return r.subscription, r.operational_intent_references, r def _setup_case(self): self.begin_test_case("Setup") @@ -305,7 +385,7 @@ def _clean_workspace(self): self._dss, extents, ) - for sub_id in self._sub_ids: + for sub_id in self._background_sub_ids: test_step_fragments.cleanup_sub(self, self._dss, sub_id) def cleanup(self): diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md index ed52cfd8b1..eadb528f43 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md @@ -87,7 +87,7 @@ Query the created subscription at every DSS provided in `dss_instances`. Confirm that the subscription that was just created is properly synchronized across all DSS instances. -#### [Get subscription](../fragments/sub/crud/read.md) +#### [Get subscription](../fragments/sub/crud/read_correct.md) Confirms that each DSS provides access to the created subscription, @@ -128,7 +128,7 @@ Query the updated subscription at every DSS provided in `dss_instances`. Confirm that the subscription that was just mutated is properly synchronized across all DSS instances. -#### [Get subscription](../fragments/sub/crud/read.md) +#### [Get subscription](../fragments/sub/crud/read_correct.md) Confirms that the subscription that was just mutated can be retrieved from any DSS. @@ -185,7 +185,7 @@ Note that this step is repeated for every secondary DSS instance. Confirm that the subscription that was just mutated is properly synchronized across all DSS instances. -#### [Get subscription](../fragments/sub/crud/read.md) +#### [Get subscription](../fragments/sub/crud/read_correct.md) Confirms that the subscription that was just mutated can be retrieved from any DSS, and that it has the expected content. diff --git a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md index 9876becfbc..9d1b87c15c 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md @@ -26,7 +26,7 @@