diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/subscription_params.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/subscription_params.py index f02485d706..be368afc80 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/subscription_params.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/subscription_params.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime from typing import List, Optional, Self @@ -72,3 +74,19 @@ def to_upsert_subscription_params( min_alt_m=self.min_alt_m, max_alt_m=self.max_alt_m, ) + + def shift_time(self, shift: datetime.timedelta) -> SubscriptionParams: + """ + Returns a new SubscriptionParams object with the start and end times shifted by the given timedelta. + """ + return SubscriptionParams( + sub_id=self.sub_id, + area_vertices=self.area_vertices, + min_alt_m=self.min_alt_m, + max_alt_m=self.max_alt_m, + start_time=self.start_time + shift, + end_time=self.end_time + shift, + base_url=self.base_url, + notify_for_op_intents=self.notify_for_op_intents, + notify_for_constraints=self.notify_for_constraints, + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md index 46c4d64888..8b981155e3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.md @@ -74,10 +74,10 @@ Verify that the subscription returned by every DSS is correctly formatted and co Verify that the version of the subscription returned by every DSS is as expected. -### Mutate subscription test step +### Mutate subscription broadcast test step -This test step mutates the previously created subscription to verify that the DSS reacts properly: notably, it checks that the subscription version is updated, -including for changes that are not directly visible, such as changing the subscription's footprint. +This test step mutates the previously created subscription, by accessing the primary DSS, to verify that the update is propagated to all other DSSes. +Notably, it checks that the subscription version is updated, including for changes that are not directly visible, such as changing the subscription's footprint. #### [Update subscription](../fragments/sub/crud/update.md) @@ -122,9 +122,51 @@ Verify that the subscription returned by every DSS is correctly formatted and co Verify that the version of the subscription returned by every DSS is as expected. -### Delete subscription test step +### Mutate subscription on secondaries test step -Attempt to delete the subscription in various ways and ensure that the DSS reacts properly. +This test step attempts to mutate the subscription on every secondary DSS instance (that is, instances through which the subscription has not been created) to confirm that such mutations are properly propagated to every DSS. + +#### 🛑 Subscription can be mutated on secondary DSS check + +If the secondary DSS does not allow the subscription to be mutated, either the secondary DSS or the primary DSS are in violation of one or both of the following requirements: + +**[astm.f3548.v21.DSS0210,1b](../../../../../requirements/astm/f3548/v21.md)**, if the `manager` of the subscription fails to be taken into account (either because the primary DSS did not propagated it, or because the secondary failed to consider it); +**[astm.f3548.v21.DSS0005,5](../../../../../requirements/astm/f3548/v21.md)**, if the secondary DSS fails to properly implement the API to mutate subscriptions. + +#### 🛑 Subscription returned by a secondary DSS is valid and correct check + +When queried for a subscription that was created via another DSS, a DSS instance is expected to provide a valid subscription. + +If it does not, it might be in violation of **[astm.f3548.v21.DSS0005,5](../../../../../requirements/astm/f3548/v21.md)**. + +#### [Update subscription](../fragments/sub/crud/update.md) + +Confirm that the secondary DSS handles the update properly. + +#### [Subscription is synchronized](../fragments/sub/sync.md) + +Confirm that the subscription that was just mutated is properly synchronized across all DSS instances. + +#### [Get subscription](../fragments/sub/crud/read.md) + +Confirms that the subscription that was just mutated can be retrieved from any DSS, and that it has the expected content. + +#### [Validate subscription](../fragments/sub/validate/correctness.md) + +Verify that the subscription returned by the DSS is properly formatted and contains the correct content. + +#### [Validate version is updated by mutation](../fragments/sub/validate/mutated.md) + +Verify that the version of the subscription returned by the DSS the subscription was mutated through has been updated. + +#### [Validate new version is synced](../fragments/sub/validate/non_mutated.md) + +Verify that the new version of the subscription has been propagated. + +### Delete subscription on primary test step + +Attempt to delete the subscription that was created on the primary DSS through the primary DSS in various ways, +and ensure that the DSS reacts properly. This also checks that the subscription data returned by a successful deletion is correct. @@ -144,7 +186,33 @@ Verify that the version of the subscription returned by the DSS is as expected Attempt to query and search for the deleted subscription in various ways -#### 🛑 Secondary DSS should not return the deleted subscription check +#### 🛑 DSS should not return the deleted subscription check + +If a DSS returns a subscription that was previously successfully deleted from the primary DSS, +either one of the primary DSS or the DSS that returned the subscription is in violation of one of the following requirements: + +**[astm.f3548.v21.DSS0210,1a](../../../../../requirements/astm/f3548/v21.md)**, if the API is not working as described by the OpenAPI specification; +**[astm.f3548.v21.DSS0215](../../../../../requirements/astm/f3548/v21.md)**, if the DSS through which the subscription was deleted is returning API calls to the client before having updated its underlying distributed storage. + +As a result, the DSS pool under test is failing to meet **[astm.f3548.v21.DSS0020](../../../../../requirements/astm/f3548/v21.md)**. + +### Delete subscriptions on secondaries test step + +Attempt to delete subscriptions that were created through the primary DSS via the secondary DSS instances. + +#### [Delete subscription](../fragments/sub/crud/delete.md) + +Confirms that a subscription can be deleted from a secondary DSS + +#### [Validate subscription](../fragments/sub/validate/correctness.md) + +Verify that the subscription returned by the DSS via the deletion is properly formatted and contains the correct content. + +#### [Validate version](../fragments/sub/validate/non_mutated.md) + +Verify that the version of the subscription returned by the DSS is as expected + +#### 🛑 DSS should not return the deleted subscription check If a DSS returns a subscription that was previously successfully deleted from the primary DSS, either one of the primary DSS or the DSS that returned the subscription is in violation of one of the following requirements: diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py index 523f516f83..d7c0b9d9a1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/subscription_synchronization.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from typing import List, Optional +from typing import List, Optional, Dict import loguru from uas_standards.astm.f3548.v21.api import Subscription, SubscriptionID @@ -35,11 +35,9 @@ class SubscriptionSynchronization(TestScenario): A scenario that checks if multiple DSS instances properly synchronize created, updated or deleted entities between them. - Not in the scope of the first version of this: + Not in the scope of the current version: - access rights (making sure only the manager of the subscription can mutate it) - - control of the area synchronization (by doing area searches against the secondaries) - - mutation of a subscription on a secondary DSS when it was created on the primary - - deletion of a subscription on a secondary DSS when it was created on the primary + """ SUB_TYPE = register_resource_type(379, "Subscription") @@ -51,12 +49,18 @@ class SubscriptionSynchronization(TestScenario): # Base identifier for the subscriptions that will be created _sub_id: SubscriptionID + # Extra sub IDs for testing only deletions + _ids_for_deletion: List[SubscriptionID] + # Base parameters used for subscription creation _sub_params: SubscriptionParams # Keep track of the current subscription state _current_subscription = Optional[Subscription] + # For the secondary deletion test + _subs_for_deletion: Dict[SubscriptionID, Subscription] + def __init__( self, dss: DSSInstanceResource, @@ -70,7 +74,6 @@ def __init__( id_generator: will let us generate specific identifiers planning_area: An Area to use for the tests. It should be an area for which the DSS is responsible, but has no other requirements. - problematically_big_area: An area that is too big to be searched for on the DSS """ super().__init__() scopes_primary = { @@ -87,6 +90,17 @@ def __init__( ] self._sub_id = id_generator.id_factory.make_id(self.SUB_TYPE) + + # For every secondary DSS, have an extra sub ID for testing deletion at each DSS + # TODO confirm that we can have as many SCD subscriptions as we want (RID limits them to 10 per area) + # TODO IDGenerators may encode the subject/identity of the participant being tested into the ID, + # therefore we may want to consider having a separate generator per DSS instance, + # or at least one per participant + self._ids_for_deletion = [ + f"{self._sub_id[:-3]}{i:03d}" + for i in range(1, len(self._dss_read_instances) + 1) + ] + self._planning_area = planning_area.specification # Build a ready-to-use 4D volume with no specified time for searching @@ -140,27 +154,35 @@ def run(self, context: ExecutionContext): self.begin_test_case("Subscription Synchronization") self.begin_test_step("Create subscription validation") - self._create_sub_with_params(self._sub_params) + self._step_create_subscriptions() self.end_test_step() self.begin_test_step("Query newly created subscription") self._query_secondaries_and_compare(self._sub_params) self.end_test_step() - self.begin_test_step("Mutate subscription") - self._test_mutate_subscriptions_shift_time() + self.begin_test_step("Mutate subscription broadcast") + self._step_mutate_subscriptions_broadcast_shift_time() self.end_test_step() self.begin_test_step("Query updated subscription") self._query_secondaries_and_compare(self._sub_params) self.end_test_step() - self.begin_test_step("Delete subscription") - self._test_delete_sub() + self.begin_test_step("Mutate subscription on secondaries") + self._step_mutate_subscriptions_secondaries_shift_time() + self.end_test_step() + + self.begin_test_step("Delete subscription on primary") + self._step_delete_sub() self.end_test_step() self.begin_test_step("Query deleted subscription") - self._test_get_deleted_sub() + self._step_get_deleted_sub() + self.end_test_step() + + self.begin_test_step("Delete subscriptions on secondaries") + self._step_delete_subscriptions_on_secondaries() self.end_test_step() self.end_test_case() @@ -171,6 +193,8 @@ def _setup_case(self): # Multiple runs of the scenario seem to rely on the same instance of it: # thus we need to reset the state of the scenario before running it. self._current_subscription = None + self._subs_for_deletion = {} + self._subs_for_deletion_params = {} self._ensure_clean_workspace_step() self.end_test_case() @@ -184,6 +208,8 @@ def _ensure_clean_workspace_step(self): def _ensure_test_sub_ids_do_not_exist(self): test_step_fragments.cleanup_sub(self, self._dss, self._sub_id) + for sub_id in self._ids_for_deletion: + test_step_fragments.cleanup_sub(self, self._dss, sub_id) def _ensure_no_active_subs_exist(self): test_step_fragments.cleanup_active_subs( @@ -192,7 +218,21 @@ def _ensure_no_active_subs_exist(self): self._planning_area_volume4d, ) - def _create_sub_with_params(self, creation_params: SubscriptionParams): + def _step_create_subscriptions(self): + # Create the 'main' test subscription: + self._current_subscription = self._create_sub_with_params(self._sub_params) + + # Create the extra subscriptions for testing deletion on secondaries at the end of the scenario + for sub_id in self._ids_for_deletion: + params = self._sub_params.copy() + params.sub_id = sub_id + extra_sub = self._create_sub_with_params(params) + self._subs_for_deletion[sub_id] = extra_sub + self._subs_for_deletion_params[sub_id] = params + + def _create_sub_with_params( + self, creation_params: SubscriptionParams + ) -> Subscription: # TODO migrate to the try/except pattern for queries newly_created = self._dss.upsert_subscription( @@ -221,8 +261,7 @@ def _create_sub_with_params(self, creation_params: SubscriptionParams): creation_params, ).validate_created_subscription(creation_params.sub_id, newly_created) - # Store the subscription - self._current_subscription = newly_created.subscription + return newly_created.subscription def _query_secondaries_and_compare(self, expected_sub_params: SubscriptionParams): for secondary_dss in self._dss_read_instances: @@ -508,28 +547,27 @@ def _compare_upsert_resp_with_params( is_implicit=False, ) - def _test_mutate_subscriptions_shift_time(self): - """Mutate the subscription by adding 10 seconds to its start and end times""" - - op = self._sub_params - sub = self._current_subscription - new_params = SubscriptionParams( - sub_id=self._sub_id, - area_vertices=op.area_vertices, - min_alt_m=op.min_alt_m, - max_alt_m=op.max_alt_m, - start_time=sub.time_start.value.datetime + timedelta(seconds=10), - end_time=sub.time_end.value.datetime + timedelta(seconds=10), - base_url=op.base_url, - notify_for_op_intents=op.notify_for_op_intents, - notify_for_constraints=op.notify_for_constraints, - ) - mutated_sub_response = self._dss.upsert_subscription( - version=sub.version, - **new_params, + def _mutate_subscription_with_dss( + self, + dss_instance: DSSInstance, + new_params: SubscriptionParams, + is_primary: bool, + ): + """ + Mutate the subscription on the given DSS instance using the given parameters. + Also updates the internal state of the scenario to reflect the new subscription. + """ + check = ( + "Subscription can be mutated" + if is_primary + else "Subscription can be mutated on secondary DSS" ) - self.record_query(mutated_sub_response) - with self.check("Subscription can be mutated", [self._primary_pid]) as check: + with self.check(check, [self._primary_pid]) as check: + mutated_sub_response = dss_instance.upsert_subscription( + version=self._current_subscription.version, + **new_params, + ) + self.record_query(mutated_sub_response) if mutated_sub_response.status_code != 200: check.record_failed( "Subscription mutation failed", @@ -546,45 +584,127 @@ def _test_mutate_subscriptions_shift_time(self): # Update the parameters we used for that subscription self._sub_params = new_params - def _test_delete_sub(self): - deleted_sub = self._dss.delete_subscription( - sub_id=self._sub_id, sub_version=self._current_subscription.version - ) - self.record_query(deleted_sub) - with self.check("Subscription can be deleted", [self._primary_pid]) as check: + def _step_mutate_subscriptions_broadcast_shift_time(self): + """Mutate the subscription on the primary DSS by adding 10 seconds to its start and end times""" + + sp = self._sub_params + new_params = sp.shift_time(timedelta(seconds=10)) + self._mutate_subscription_with_dss(self._dss, new_params, is_primary=True) + + def _step_mutate_subscriptions_secondaries_shift_time(self): + """Mutate the subscription on every secondary DSS by adding 10 seconds to its start and end times, + then checking on every DSS that the response is valid and corresponds to the expected parameters.""" + + for secondary_dss in self._dss_read_instances: + # Mutate the subscription on the secondary DSS + self._mutate_subscription_with_dss( + secondary_dss, + self._sub_params.shift_time(timedelta(seconds=10)), + is_primary=False, + ) + # Check that the mutation was propagated to every DSS: + self._query_secondaries_and_compare(self._sub_params) + + def _delete_sub_from_dss( + self, + dss_instance: DSSInstance, + sub_id: str, + version: str, + expected_params: SubscriptionParams, + ) -> bool: + """ + Delete the subscription on the given DSS instance using the given parameters. + Returns True if the subscription was successfully deleted, False otherwise. + """ + with self.check( + "Subscription can be deleted", [dss_instance.participant_id] + ) as check: + deleted_sub = dss_instance.delete_subscription(sub_id, version) + self.record_query(deleted_sub) if deleted_sub.status_code != 200: check.record_failed( "Subscription deletion failed", details=f"Subscription deletion failed with status code {deleted_sub.status_code}", query_timestamps=[deleted_sub.request.timestamp], ) + return False with self.check( - "Delete subscription response format conforms to spec", [self._primary_pid] + "Delete subscription response format conforms to spec", + [dss_instance.participant_id], ) as check: SubscriptionValidator( check, self, - [self._primary_pid], - self._sub_params, + [dss_instance.participant_id], + expected_params, ).validate_deleted_subscription( - expected_sub_id=self._sub_id, + expected_sub_id=sub_id, deleted_subscription=deleted_sub, - expected_version=self._current_subscription.version, + expected_version=version, is_implicit=False, ) - self._current_subscription = None + return True - def _test_get_deleted_sub(self): + def _step_delete_sub(self): + if self._delete_sub_from_dss( + self._dss, + self._sub_id, + self._current_subscription.version, + self._sub_params, + ): + self._current_subscription = None + + def _step_delete_subscriptions_on_secondaries(self): + # Pair a sub ID to delete together with a secondary DSS + for sub_id, secondary_dss in zip( + self._ids_for_deletion, self._dss_read_instances + ): + # Delete the subscription on the secondary DSS + if not self._delete_sub_from_dss( + secondary_dss, + sub_id, + self._subs_for_deletion[sub_id].version, + self._subs_for_deletion_params[sub_id], + ): + # If the deletion failed but the scenario has not terminated, we end this step here. + return + # Check that the primary knows about the deletion: + self._confirm_dss_has_no_sub( + self._dss, sub_id, secondary_dss.participant_id + ) + # Check that the deletion was propagated to every DSS: + self._confirm_no_secondary_has_sub(sub_id, secondary_dss.participant_id) + + def _step_get_deleted_sub(self): + self._confirm_no_secondary_has_sub(self._sub_id, self._dss.participant_id) + + def _confirm_no_secondary_has_sub( + self, sub_id: str, deleted_on_participant_id: str + ): + """Confirm that no secondary DSS has the subscription. + deleted_on_participant_id specifies the participant_id of the DSS where the subscription was deleted.""" for secondary_dss in self._dss_read_instances: - self._confirm_secondary_has_no_sub(secondary_dss) + self._confirm_dss_has_no_sub( + secondary_dss, sub_id, deleted_on_participant_id + ) - def _confirm_secondary_has_no_sub(self, secondary_dss: DSSInstance): - fetched_sub = secondary_dss.get_subscription(self._sub_id) + def _confirm_dss_has_no_sub( + self, + dss_instance: DSSInstance, + sub_id: str, + other_participant_id: Optional[str], + ): + """Confirm that a DSS has no subscription. + other_participant_id may be specified if a failed check may be caused by it.""" + participants = [dss_instance.participant_id] + if other_participant_id: + participants.append(other_participant_id) + fetched_sub = dss_instance.get_subscription(sub_id) with self.check( - "Secondary DSS should not return the deleted subscription", - [secondary_dss.participant_id, self._primary_pid], + "DSS should not return the deleted subscription", + participants, ) as check: if fetched_sub.status_code != 404: check.record_failed( diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py index 1cf30767cf..fab8fe60ba 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py @@ -51,7 +51,7 @@ def cleanup_sub( if existing_sub.status_code not in [200, 404]: check.record_failed( summary=f"Could not query subscription {sub_id}", - details=f"When attempting to query subscription {sub_id} from the DSS, received {existing_sub.status_code}", + details=f"When attempting to query subscription {sub_id} from the DSS, received {existing_sub.status_code}: {existing_sub.error_message}", query_timestamps=[existing_sub.request.timestamp], ) diff --git a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md index f4d40f81d0..aafcf4ab03 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md @@ -22,7 +22,7 @@