diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 134bc4327f..6432f00c5d 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -435,6 +435,19 @@ def error_message(self) -> Optional[str]: else None ) + @property + def failure_details(self) -> Optional[str]: + """ + Returns the error message if one is available, otherwise returns the response content. + To be used to fill in the details of a check failure. + Note that 'failure' here is context dependent: possibly a 401 is expected and a 404 or 200 is returned, + in both situations we would like to return the most relevant information. + """ + err_msg = self.error_message + if err_msg: + return err_msg + return self.response.json + def get_client_sub(self): headers = self.request.headers if "Authorization" in headers: diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md index a661a3a047..171b6d3b57 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.md @@ -38,7 +38,11 @@ Optional scopes that will allow the scenario to provide additional coverage: ### [Ensure clean workspace test step](../clean_workspace.md) -This step ensures that no entity with the known test IDs exists in the DSS. +This step ensures that the availability for the test identifier is set to `Unknown`. + +#### [Availability can be requested](../fragments/availability/read.md) + +#### [Availability can be set](../fragments/availability/update.md) ## Endpoint authorization test case @@ -313,6 +317,77 @@ it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../require If the DSS does not allow searching for operational intents when valid credentials are presented, it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../requirements/astm/f3548/v21.md)**. +### Availability endpoints authentication test step + +#### 🛑 Unauthorized requests return the proper error message body check + +If the DSS under test does not return a proper error message body when an unauthorized request is received, +it fails to properly implement the OpenAPI specification that is part of **[astm.f3548.v21.DSS0100,1](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Read availability with missing credentials check + +If the DSS under test allows the fetching of a USS's availability without any credentials being presented, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Read availability with invalid credentials check + +If the DSS under test allows the fetching of a USS's availability with credentials that are well-formed but invalid, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Read availability with missing scope check + +If the DSS under test allows the fetching of a USS's availability with valid credentials but a missing scope, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Read availability with incorrect scope check + +If the DSS under test allows the fetching of a USS's availability with valid credentials but an incorrect scope, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Read availability with valid credentials check + +If the DSS does not allow fetching a USS's availability when valid credentials are presented, +it is in violation of **[astm.f3548.v21.DSS0100,1](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 USS Availability Get response format conforms to spec check + +The response to a successful USS Availability request is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21, +otherwise, the DSS is failing to implement **[astm.f3548.v21.DSS0100,1](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Set availability with missing credentials check + +If the DSS under test allows the setting of a USS's availability without any credentials being presented, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Set availability with invalid credentials check + +If the DSS under test allows the setting of a USS's availability with credentials that are well-formed but invalid, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Set availability with missing scope check + +If the DSS under test allows the setting of a USS's availability with valid credentials but a missing scope, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Set availability with incorrect scope check + +If the DSS under test allows the setting of a USS's availability with valid credentials but an incorrect scope, +it is in violation of **[astm.f3548.v21.DSS0210,A2-7-2,7](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 Set availability with valid credentials check + +If the DSS does not allow setting a USS's availability when valid credentials are presented, +it is in violation of **[astm.f3548.v21.DSS0100,1](../../../../../requirements/astm/f3548/v21.md)**. + +#### 🛑 USS Availability Set response format conforms to spec check + +The response to a successful USS Availability Set request is expected to conform to the format defined by the OpenAPI specification under the `A3.1` Annex of ASTM F3548−21, +otherwise, the DSS is failing to implement **[astm.f3548.v21.DSS0100,1](../../../../../requirements/astm/f3548/v21.md)**. + ## [Cleanup](../clean_workspace.md) +### [Availability can be requested](../fragments/availability/read.md) + +### [Availability can be set](../fragments/availability/update.md) + The cleanup phase of this test scenario removes the subscription with the known test ID if it has not been removed before. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py index 6afa6324ae..868d5f0685 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/authentication_validation.py @@ -1,5 +1,8 @@ from datetime import datetime, timedelta +from uas_standards.astm.f3548.v21.api import UssAvailabilityState + +from monitoring.monitorlib.fetch import QueryError from monitoring.uss_qualifier.resources.astm.f3548.v21.subscription_params import ( SubscriptionParams, ) @@ -11,12 +14,18 @@ from monitoring.monitorlib.geotemporal import Volume4D from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.prober.infrastructure import register_resource_type -from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstanceResource +from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import ( + DSSInstanceResource, + DSSInstance, +) from monitoring.uss_qualifier.resources.astm.f3548.v21.planning_area import ( PlanningAreaResource, ) from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.scenarios.astm.utm.dss import test_step_fragments +from monitoring.uss_qualifier.scenarios.astm.utm.dss.authentication.availability_api_validator import ( + AvailabilityAuthValidator, +) from monitoring.uss_qualifier.scenarios.astm.utm.dss.authentication.generic import ( GenericAuthValidator, ) @@ -52,8 +61,16 @@ class AuthenticationValidation(TestScenario): _sub_validator: SubscriptionAuthValidator _oir_validator: OperationalIntentRefAuthValidator + _availability_validator: AvailabilityAuthValidator + _sub_params: SubscriptionParams + _scd_dss: DSSInstance + _availability_dss: DSSInstance + + _wrong_scope_for_availability: Scope + _wrong_scope_for_scd: Scope + def __init__( self, dss: DSSInstanceResource, @@ -88,6 +105,24 @@ def __init__( self._wrong_scope_for_scd ] = "Attempt to query subscriptions with wrong scope" + availability_scopes = { + Scope.AvailabilityArbitration: "read and set availability for a USS" + } + + self._wrong_scope_for_availability = dss.get_authorized_scope_not_in( + [ + Scope.AvailabilityArbitration, # Allowed to get and update + Scope.ConformanceMonitoringForSituationalAwareness, # Allowed to get + Scope.StrategicCoordination, # Allowed to get + "", + ] + ) + + if self._wrong_scope_for_availability is not None: + availability_scopes[ + self._wrong_scope_for_availability + ] = "Attempt to query availability with wrong scope" + self._test_missing_scope = False if dss.can_use_scope(""): scd_scopes[""] = "Attempt to query subscriptions with missing scope" @@ -96,6 +131,7 @@ def __init__( # Note: .get_instance should be called once we know every scope we will need, # in order to guarantee that they are indeed available. self._scd_dss = dss.get_instance(scd_scopes) + self._availability_dss = dss.get_instance(availability_scopes) self._pid = [dss.participant_id] self._test_id = id_generator.id_factory.make_id(self.SUB_TYPE) @@ -146,6 +182,19 @@ def run(self, context: ExecutionContext): test_missing_scope=self._test_missing_scope, ) + self._availability_validator = AvailabilityAuthValidator( + scenario=self, + generic_validator=GenericAuthValidator( + self, self._availability_dss, Scope.AvailabilityArbitration + ), + dss=self._availability_dss, + test_id=self._test_id, + no_auth_session=self._no_auth_session, + invalid_token_session=self._invalid_token_session, + test_wrong_scope=self._wrong_scope_for_availability, + test_missing_scope=self._test_missing_scope, + ) + self._sub_params = self._planning_area.get_new_subscription_params( subscription_id=self._test_id, # Set this slightly in the past: we will update the subscriptions @@ -171,6 +220,17 @@ def run(self, context: ExecutionContext): "wrong_scope_scd", "Incorrect scope testing disabled for SCD endpoints" ) + if self._wrong_scope_for_availability: + self.record_note( + "wrong_scope_availability", + f"Incorrect scope testing enabled for availability endpoints with scope {self._wrong_scope_for_availability}.", + ) + else: + self.record_note( + "wrong_scope_availability", + "Incorrect scope testing disabled for availability endpoints", + ) + if self._test_missing_scope: self.record_note("missing_scope", "Missing scope testing enabled.") else: @@ -185,6 +245,10 @@ def run(self, context: ExecutionContext): self._oir_validator.verify_oir_endpoints_authentication() self.end_test_step() + self.begin_test_step("Availability endpoints authentication") + self._availability_validator.verify_availability_endpoints_authentication() + self.end_test_step() + self.end_test_case() self.end_test_scenario() @@ -210,6 +274,9 @@ def _ensure_test_entities_dont_exist(self): test_step_fragments.cleanup_op_intent(self, self._scd_dss, self._test_id) test_step_fragments.cleanup_sub(self, self._scd_dss, self._test_id) + # Make sure the test ID for uss availability is set to 'Unknown' + self._ensure_availability_is_unknown() + def _ensure_no_active_subs_exist(self): test_step_fragments.cleanup_active_subs( self, @@ -217,6 +284,37 @@ def _ensure_no_active_subs_exist(self): self._planning_area_volume4d, ) + def _ensure_availability_is_unknown(self): + + with self.check("USS Availability can be requested", self._pid) as check: + try: + availability, q = self._availability_dss.get_uss_availability( + self._test_id, scope=Scope.AvailabilityArbitration + ) + self.record_query(q) + except QueryError as e: + self.record_queries(e.queries) + check.record_failed( + summary="Could not get USS availability", + details=f"Failed to get USS availability: {e}", + query_timestamps=[q.request.timestamp for q in e.queries], + ) + + if availability.status != UssAvailabilityState.Unknown: + with self.check("USS Availability can be updated", self._pid) as check: + try: + availability, q = self._availability_dss.set_uss_availability( + self._test_id, available=None, version=availability.version + ) + self.record_query(q) + except QueryError as e: + self.record_queries(e.queries) + check.record_failed( + summary="Could not set USS availability", + details=f"Failed to set USS availability: {e}", + query_timestamps=[q.request.timestamp for q in e.queries], + ) + def cleanup(self): self.begin_cleanup() self._ensure_test_entities_dont_exist() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/availability_api_validator.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/availability_api_validator.py new file mode 100644 index 0000000000..f065f94309 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/authentication/availability_api_validator.py @@ -0,0 +1,301 @@ +from typing import Optional + +from implicitdict import ImplicitDict +from uas_standards.astm.f3548.v21.api import ( + OPERATIONS, + OperationID, + UssAvailabilityStatusResponse, + SetUssAvailabilityStatusParameters, + UssAvailabilityState, +) +from uas_standards.astm.f3548.v21.constants import Scope + +from monitoring.monitorlib.fetch import QueryType, QueryError +from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance +from monitoring.uss_qualifier.scenarios.astm.utm.dss.authentication.generic import ( + GenericAuthValidator, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario, PendingCheck + + +class AvailabilityAuthValidator: + + _current_availability: Optional[UssAvailabilityStatusResponse] = None + + def __init__( + self, + scenario: TestScenario, + generic_validator: GenericAuthValidator, + dss: DSSInstance, + test_id: str, + no_auth_session: UTMClientSession, + invalid_token_session: UTMClientSession, + test_wrong_scope: Optional[str] = None, + test_missing_scope: bool = False, + ): + """ + + Args: + scenario: Scenario on which the checks will be done + generic_validator: Provides generic verification methods for DSS API calls + dss: the DSS instance being tested + test_id: identifier to use for the subscriptions that will be created + no_auth_session: an unauthenticated session + invalid_token_session: a session using a well-formed token that has an invalid signature + test_wrong_scope: a valid scope that is not allowed to perform operations on subscriptions, if available. + If None, checks using a wrong scope will be skipped. + test_missing_scope: if True, will attempt to perform operations without specifying a scope using the valid credentials. + """ + self._scenario = scenario + self._gen_val = generic_validator + self._dss = dss + self._pid = dss.participant_id + self._test_id = test_id + + self._no_auth_session = no_auth_session + self._invalid_token_session = invalid_token_session + + self._test_wrong_scope = test_wrong_scope + self._test_missing_scope = test_missing_scope + + def verify_availability_endpoints_authentication(self): + self._verify_read() + self._verify_set() + + def _verify_read(self): + op = OPERATIONS[OperationID.GetUssAvailability] + query_kwargs = dict( + verb=op.verb, + url=op.path.format(uss_id=self._test_id), + query_type=QueryType.F3548v21DSSGetUssAvailability, + participant_id=self._dss.participant_id, + ) + + # No auth: + no_auth_q = self._gen_val.query_no_auth(**query_kwargs) + with self._scenario.check( + "Read availability with missing credentials", self._pid + ) as check: + if no_auth_q.status_code != 401: + check.record_failed( + summary=f"Expected 401, got {no_auth_q.status_code}", + details=no_auth_q.failure_details, + query_timestamps=[no_auth_q.request.timestamp], + ) + + self._gen_val.verify_4xx_response(no_auth_q) + + # Bad token signature: + invalid_token_q = self._gen_val.query_invalid_token(**query_kwargs) + with self._scenario.check( + "Read availability with invalid credentials", self._pid + ) as check: + if invalid_token_q.status_code != 401: + check.record_failed( + summary=f"Expected 401, got {invalid_token_q.status_code}", + details=invalid_token_q.failure_details, + query_timestamps=[invalid_token_q.request.timestamp], + ) + + self._gen_val.verify_4xx_response(invalid_token_q) + + # Valid credentials but missing scope: + if self._test_missing_scope: + no_scope_q = self._gen_val.query_missing_scope(**query_kwargs) + with self._scenario.check( + "Read availability with missing scope", self._pid + ) as check: + if no_scope_q.status_code != 401: + check.record_failed( + summary=f"Expected 401, got {no_scope_q.status_code}", + details=no_scope_q.failure_details, + query_timestamps=[no_scope_q.request.timestamp], + ) + + self._gen_val.verify_4xx_response(no_scope_q) + + # Valid credentials but wrong scope: + if self._test_wrong_scope: + wrong_scope_q = self._gen_val.query_wrong_scope( + scope=self._test_wrong_scope, **query_kwargs + ) + with self._scenario.check( + "Read availability with incorrect scope", self._pid + ) as check: + if wrong_scope_q.status_code != 403: + check.record_failed( + summary=f"Expected 403, got {wrong_scope_q.status_code}", + details=wrong_scope_q.failure_details, + query_timestamps=[wrong_scope_q.request.timestamp], + ) + + self._gen_val.verify_4xx_response(wrong_scope_q) + + # Correct token: + # - confirms that the request would otherwise work + request_ok = self._gen_val.query_valid_auth(**query_kwargs) + with self._scenario.check( + "Read availability with valid credentials", self._pid + ) as check: + if request_ok.status_code != 200: + check.record_failed( + summary=f"Expected 200, got {request_ok.status_code}", + details=request_ok.failure_details, + query_timestamps=[request_ok.request.timestamp], + ) + + with self._scenario.check( + "USS Availability Get response format conforms to spec", self._pid + ) as check: + try: + parsed_resp = request_ok.parse_json_result( + UssAvailabilityStatusResponse + ) + except QueryError as e: + check.record_failed( + summary="Could not parse the response body", + details=f"Could not parse the response body as a UssAvailabilityStatusResponse: {e}", + query_timestamps=[request_ok.request.timestamp], + ) + + self._current_availability = parsed_resp + + def _verify_set(self): + op = OPERATIONS[OperationID.SetUssAvailability] + query_kwargs = dict( + verb=op.verb, + url=op.path.format(uss_id=self._test_id), + json=SetUssAvailabilityStatusParameters( + old_version=self._current_availability.version, + availability=UssAvailabilityState.Down, + ), + query_type=QueryType.F3548v21DSSSetUssAvailability, + participant_id=self._dss.participant_id, + ) + + # No auth: + no_auth_q = self._gen_val.query_no_auth(**query_kwargs) + with self._scenario.check( + "Set availability with missing credentials", self._pid + ) as check: + if no_auth_q.status_code != 401: + check.record_failed( + summary=f"Expected 401, got {no_auth_q.status_code}", + details=no_auth_q.failure_details, + query_timestamps=[no_auth_q.request.timestamp], + ) + + self._sanity_check_availability_not_updated( + check, self._current_availability + ) + + self._gen_val.verify_4xx_response(no_auth_q) + + # Bad token signature: + invalid_token_q = self._gen_val.query_invalid_token(**query_kwargs) + with self._scenario.check( + "Set availability with invalid credentials", self._pid + ) as check: + if invalid_token_q.status_code != 401: + check.record_failed( + summary=f"Expected 401, got {invalid_token_q.status_code}", + details=invalid_token_q.failure_details, + query_timestamps=[invalid_token_q.request.timestamp], + ) + + self._sanity_check_availability_not_updated( + check, self._current_availability + ) + + self._gen_val.verify_4xx_response(invalid_token_q) + + # Valid credentials but missing scope: + if self._test_missing_scope: + no_scope_q = self._gen_val.query_missing_scope(**query_kwargs) + with self._scenario.check( + "Set availability with missing scope", self._pid + ) as check: + if no_scope_q.status_code != 401: + check.record_failed( + summary=f"Expected 401, got {no_scope_q.status_code}", + details=no_scope_q.failure_details, + query_timestamps=[no_scope_q.request.timestamp], + ) + + self._sanity_check_availability_not_updated( + check, self._current_availability + ) + self._gen_val.verify_4xx_response(no_scope_q) + + # Valid credentials but wrong scope: + if self._test_wrong_scope: + wrong_scope_q = self._gen_val.query_wrong_scope( + scope=self._test_wrong_scope, **query_kwargs + ) + with self._scenario.check( + "Set availability with incorrect scope", self._pid + ) as check: + if wrong_scope_q.status_code != 403: + check.record_failed( + summary=f"Expected 403, got {wrong_scope_q.status_code}", + details=wrong_scope_q.failure_details, + query_timestamps=[wrong_scope_q.request.timestamp], + ) + + self._sanity_check_availability_not_updated( + check, self._current_availability + ) + self._gen_val.verify_4xx_response(wrong_scope_q) + + # Correct token: + # - confirms that the request would otherwise work + request_ok = self._gen_val.query_valid_auth(**query_kwargs) + with self._scenario.check( + "Set availability with valid credentials", self._pid + ) as check: + if request_ok.status_code != 200: + check.record_failed( + summary=f"Expected 200, got {request_ok.status_code}", + details=request_ok.failure_details, + query_timestamps=[request_ok.request.timestamp], + ) + + with self._scenario.check( + "USS Availability Set response format conforms to spec", self._pid + ) as check: + try: + _ = request_ok.parse_json_result(UssAvailabilityStatusResponse) + except QueryError as e: + check.record_failed( + summary="Could not parse the response body", + details=f"Could not parse the response body as a UssAvailabilityStatusResponse: {e}", + query_timestamps=[request_ok.request.timestamp], + ) + + def _sanity_check_availability_not_updated( + self, sanity_check: PendingCheck, expected: UssAvailabilityStatusResponse + ): + with self._scenario.check( + "Read availability with valid credentials", self._pid + ) as query_check: + try: + response, q = self._dss.get_uss_availability( + self._test_id, scope=Scope.AvailabilityArbitration + ) + self._scenario.record_query(q) + except QueryError as e: + self._scenario.record_queries(e.queries) + query_check.record_failed( + summary="Failed to query USS availability to determine if it was updated", + details=f"Failed to query USS availability: {e}", + query_timestamps=[e.queries[0].request.timestamp], + ) + return + + if response != expected: + sanity_check.record_failed( + summary="USS availability was updated", + details=f"Expected the USS availability to remain unchanged ({expected}), but it was updated to {response}", + query_timestamps=[q.request.timestamp], + ) diff --git a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md index 5c26113f82..3d815ac1dd 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/utm/dss_probing.md @@ -57,7 +57,7 @@