diff --git a/monitoring/uss_qualifier/configurations/dev/uspace_f3548.yaml b/monitoring/uss_qualifier/configurations/dev/uspace_f3548.yaml new file mode 100644 index 0000000000..fa17269b43 --- /dev/null +++ b/monitoring/uss_qualifier/configurations/dev/uspace_f3548.yaml @@ -0,0 +1,37 @@ +# Configuration to run just the ASTM F3548-21 part of the uspace.yaml configuration +v1: + test_run: + resources: + $ref: ./uspace.yaml#/v1/test_run/resources + action: + $ref: ./uspace.yaml#/v1/test_run/action + execution: + stop_fast: true + skip_action_when: + - is_test_scenario: {} + except_when: + - has_ancestor: + which: + - is_test_suite: {} + regex_matches_name: ASTM F3548-21 + - is_test_scenario: {} + regex_matches_name: Configure mock_uss locality + - is_test_scenario: {} + regex_matches_name: Unconfigure mock_uss locality + artifacts: + $ref: ./uspace.yaml#/v1/artifacts + validation: + criteria: + - applicability: + test_scenarios: {} + pass_condition: + each_element: + has_execution_error: false + - applicability: + failed_checks: + has_severity: + higher_than: Low + pass_condition: + elements: + count: + equal_to: 0 diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md index 5ea938f54b..793df46f56 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md @@ -16,7 +16,7 @@ The flight planners subject to evaluation. In this step, all successful requests for operational intent details made to the USSs that are part of the flight planners provided as resource are used to determine and evaluate the 95th percentile of the requests durations. -#### Operational intent details requests take no more than [MaxRespondToOIDetailsRequest] second 95% of the time check +#### ⚠️ Operational intent details requests take no more than [MaxRespondToOIDetailsRequest] second 95% of the time check If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second), this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**. @@ -27,7 +27,7 @@ this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f This step verifies that interactions with the interoperability test instances happened and where at least partly successful. -#### Interoperability test instance is available check +#### ⚠️ Interoperability test instance is available check This check ensures that interactions with the interoperability test instance that each USS must provide are possible. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py index 79657c3d36..df647c714b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py @@ -4,7 +4,6 @@ from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import evaluation, QueryType -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ParticipantID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource from monitoring.uss_qualifier.scenarios.scenario import TestScenario @@ -115,7 +114,6 @@ def _op_intent_details_step(self): if p95 > constants.MaxRespondToOIDetailsRequestSeconds: check.record_failed( summary=f"95th percentile of durations for operational intent details requests to USS is higher than threshold", - severity=Severity.Medium, participants=[participant], details=f"threshold: {constants.MaxRespondToOIDetailsRequestSeconds}s, 95th percentile: {p95}s", ) @@ -174,7 +172,6 @@ def _validate_participant_test_interop_instance( if not success: check.record_failed( summary=f"No successful {query_type} interaction with interoperability test instance", - severity=Severity.Medium, details=f"Found no successful {query_type} interaction with interoperability test instance, " f"indicating that the test instance is either not available or not properly implemented.", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.md index 16ff85c363..9a1b61282e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.md @@ -20,12 +20,12 @@ A resources.astm.f3548.v21.DSSInstancesResource containing at least two DSS inst ### Test environment requirements test step -#### DSS instance is publicly addressable check +#### 🛑 DSS instance is publicly addressable check As per **[astm.f3548.v21.DSS0300](../../../requirements/astm/f3548/v21.md)** the DSS instance should be publicly addressable. As such, this check will fail if the resolved IP of the DSS host is a private IP address, unless that is explicitly expected. -#### DSS instance is reachable check +#### 🛑 DSS instance is reachable check As per **[astm.f3548.v21.DSS0300](../../../requirements/astm/f3548/v21.md)** the DSS instance should be publicly addressable. As such, this check will fail if the DSS is not reachable with a dummy query. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py index ce59ff94fe..be9b622562 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py @@ -6,7 +6,6 @@ from monitoring.uss_qualifier.suites.suite import ExecutionContext from uas_standards.astm.f3548.v21.api import Volume4D, Volume3D, Polygon, LatLngPoint -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import ( DSSInstancesResource, DSSInstanceResource, @@ -70,7 +69,6 @@ def _test_env_reqs(self): elif ipaddress.ip_address(ip_addr).is_private: check.record_failed( summary=f"DSS host {parsed_url.netloc} is not publicly addressable", - severity=Severity.Medium, participants=[dss.participant_id], details=f"DSS (URL: {dss.base_url}, netloc: {parsed_url.netloc}, resolved IP: {ip_addr}) is not publicly addressable", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md index 5acf0a836a..8469fbf59b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md @@ -213,5 +213,5 @@ original accepted request), or it should have been removed (because the USS reje ## Cleanup -### Successful flight deletion check +### ⚠️ Successful flight deletion check **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.md b/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.md index 0f61b6441e..f8e575b1b9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.md @@ -58,17 +58,17 @@ The setup will create two separate operational intents: one for each set of the ### Ensure clean workspace test step -#### Operational intents can be queried directly by their ID check +#### 🛑 Operational intents can be queried directly by their ID check If an existing operational intent cannot directly be queried by its ID, the DSS implementation is in violation of **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. -#### Operational intents can be searched using valid credentials check +#### 🛑 Operational intents can be searched using valid credentials check A client with valid credentials should be allowed to search for operational intents in a given area. Otherwise, the DSS is not in compliance with **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. -#### Operational intents can be deleted by their owner check +#### 🛑 Operational intents can be deleted by their owner check If an existing operational intent cannot be deleted when providing the proper ID and OVN, the DSS implementation is in violation of **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. @@ -79,12 +79,12 @@ This test step ensures that an operation intent created with the main credential To verify that the second credentials are valid, it will also create an operational intent with those credentials. -#### Can create an operational intent with valid credentials check +#### 🛑 Can create an operational intent with valid credentials check If the DSS does not allow the creation of operation intents when the required parameters and credentials are provided, it is in violation of **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. -#### Passed sets of credentials are different check +#### 🛑 Passed sets of credentials are different check This scenario requires two sets of credentials that have a different 'sub' claim in order to validate that the DSS properly controls access to operational intents. @@ -98,29 +98,29 @@ This test case ensures that the DSS does not allow a caller to modify or delete This test step will attempt to modify the operational intent that was created using the configured `dss` resource, using the credentials provided in the `second_utm_auth` resource, and expect all such attempts to fail. -#### Operational intents can be queried directly by their ID check +#### 🛑 Operational intents can be queried directly by their ID check If an existing operational intent cannot directly be queried by its ID, the DSS implementation is in violation of **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. -#### Non-owning credentials cannot modify operational intent check +#### 🛑 Non-owning credentials cannot modify operational intent check If an operational intent can be modified by a client which did not create it, the DSS implementation is in violation of **[astm.f3548.v21.OPIN0035](../../../requirements/astm/f3548/v21.md)**. -#### Non-owning credentials cannot delete operational intent check +#### 🛑 Non-owning credentials cannot delete operational intent check If an operational intent can be deleted by a client which did not create it, the DSS implementation is in violation of **[astm.f3548.v21.OPIN0035](../../../requirements/astm/f3548/v21.md)**. ## Cleanup -### Operational intents can be queried directly by their ID check +### 🛑 Operational intents can be queried directly by their ID check If an existing operational intent cannot directly be queried by its ID, the DSS implementation is in violation of **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. -### Operational intents can be deleted by their owner check +### 🛑 Operational intents can be deleted by their owner check If an existing operational intent cannot be deleted when providing the proper ID and OVN, the DSS implementation is in violation of **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.py b/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.py index 76b69bf214..b85adf5567 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/op_intent_access_control.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List import loguru from uas_standards.astm.f3548.v21.api import OperationalIntentState @@ -6,7 +6,6 @@ from monitoring.monitorlib.geotemporal import Volume4DCollection from uas_standards.astm.f3548.v21 import api as f3548v21 from monitoring.prober.infrastructure import register_resource_type -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import ( DSSInstance, @@ -133,8 +132,7 @@ def _clean_known_op_intents_ids(self): if q.response.status_code not in [200, 404]: check.record_failed( f"Could not access operational intent using main credentials", - Severity.High, - f"DSS responded with {q.response.status_code} to attempt to access OI {self._oid_1}", + details=f"DSS responded with {q.response.status_code} to attempt to access OI {self._oid_1}", query_timestamps=[q.request.timestamp], ) if q.response.status_code != 404: @@ -146,8 +144,7 @@ def _clean_known_op_intents_ids(self): ) as check: check.record_failed( f"Could not delete operational intent using main credentials", - Severity.High, - f"DSS responded with {dq.response.status_code} to attempt to delete OI {self._oid_1}", + details=f"DSS responded with {dq.response.status_code} to attempt to delete OI {self._oid_1}", query_timestamps=[dq.request.timestamp], ) @@ -159,8 +156,7 @@ def _clean_known_op_intents_ids(self): if q.response.status_code not in [200, 404]: check.record_failed( f"Could not access operational intent using second credentials", - Severity.High, - f"DSS responded with {q.response.status_code} to attempt to access OI {self._oid_2}", + details=f"DSS responded with {q.response.status_code} to attempt to access OI {self._oid_2}", query_timestamps=[q.request.timestamp], ) if q.response.status_code != 404: @@ -174,8 +170,7 @@ def _clean_known_op_intents_ids(self): if dq.response.status_code != 200: check.record_failed( f"Could not delete operational intent using second credentials", - Severity.High, - f"DSS responded with {dq.response.status_code} to attempt to delete OI {self._oid_2}", + details=f"DSS responded with {dq.response.status_code} to attempt to delete OI {self._oid_2}", query_timestamps=[dq.request.timestamp], ) @@ -192,8 +187,7 @@ def _ensure_clean_workspace(self): if q.response.status_code != 200: check.record_failed( f"Could not search operational intents using main credentials", - Severity.High, - f"DSS responded with {q.response.status_code} to attempt to search OIs", + details=f"DSS responded with {q.response.status_code} to attempt to search OIs", query_timestamps=[q.request.timestamp], ) @@ -208,8 +202,7 @@ def _ensure_clean_workspace(self): if dq.response.status_code != 200: check.record_failed( f"Could not delete operational intent using main credentials", - Severity.High, - f"DSS responded with {dq.response.status_code} to attempt to delete OI {op_intent.id}", + details=f"DSS responded with {dq.response.status_code} to attempt to delete OI {op_intent.id}", query_timestamps=[dq.request.timestamp], ) @@ -223,8 +216,7 @@ def _ensure_clean_workspace(self): if q.response.status_code != 200: check.record_failed( f"Could not search operational intents using second credentials", - Severity.High, - f"DSS responded with {q.response.status_code} to attempt to search OIs", + details=f"DSS responded with {q.response.status_code} to attempt to search OIs", query_timestamps=[q.request.timestamp], ) @@ -244,8 +236,7 @@ def _ensure_clean_workspace(self): if dq.response.status_code != 200: check.record_failed( f"Could not delete operational intent using second credentials", - Severity.High, - f"DSS responded with {dq.response.status_code} to attempt to delete OI {op_intent.id}", + details=f"DSS responded with {dq.response.status_code} to attempt to delete OI {op_intent.id}", query_timestamps=[dq.request.timestamp], ) @@ -265,8 +256,7 @@ def _create_op_intents(self): if q1.response.status_code != 201: check.record_failed( f"Could not create operational intent using main credentials", - Severity.High, - f"DSS responded with {q1.response.status_code} to attempt to create OI {self._oid_1}", + details=f"DSS responded with {q1.response.status_code} to attempt to create OI {self._oid_1}", query_timestamps=[q1.request.timestamp], ) @@ -288,8 +278,7 @@ def _create_op_intents(self): if q2.response.status_code != 201: check.record_failed( f"Could not create operational intent using second credentials", - Severity.High, - f"DSS responded with {q2.response.status_code} to attempt to create OI {self._oid_2}", + details=f"DSS responded with {q2.response.status_code} to attempt to create OI {self._oid_2}", query_timestamps=[q2.request.timestamp], ) @@ -306,8 +295,7 @@ def _ensure_credentials_are_different(self): ): check.record_failed( f"Second set of credentials is not different from the first", - Severity.High, - f"The same credentials were provided for the main 'dss' and the additional 'second_utm_auth'" + details=f"The same credentials were provided for the main 'dss' and the additional 'second_utm_auth'" f" resources ({self._dss.client.auth_adapter.get_sub()}),", ) @@ -329,8 +317,7 @@ def _check_mutation_on_non_owned_intent_fails(self): if q.response.status_code != 403: check.record_failed( f"Could update operational intent using second credentials", - Severity.High, - f"DSS responded with {q.response.status_code} to attempt to update OI {self._oid_1}", + details=f"DSS responded with {q.response.status_code} to attempt to update OI {self._oid_1}", query_timestamps=[q.request.timestamp], ) # Attempt to update the base_url of the intent created with the main credentials using the second credentials @@ -350,8 +337,7 @@ def _check_mutation_on_non_owned_intent_fails(self): if q.response.status_code != 403: check.record_failed( f"Could update operational intent using second credentials", - Severity.High, - f"DSS responded with {q.response.status_code} to attempt to update OI {self._oid_1}", + details=f"DSS responded with {q.response.status_code} to attempt to update OI {self._oid_1}", query_timestamps=[q.request.timestamp], ) @@ -367,8 +353,7 @@ def _check_mutation_on_non_owned_intent_fails(self): if dq.response.status_code != 403: check.record_failed( f"Could delete operational intent using second credentials", - Severity.High, - f"DSS responded with {dq.response.status_code} to attempt to delete OI {self._oid_1}", + details=f"DSS responded with {dq.response.status_code} to attempt to delete OI {self._oid_1}", query_timestamps=[dq.request.timestamp], ) @@ -382,8 +367,7 @@ def _check_mutation_on_non_owned_intent_fails(self): if qcheck.response.status_code != 200: check.record_failed( f"Could not access operational intent using main credentials", - Severity.High, - f"DSS responded with {qcheck.response.status_code} to attempt to access OI {self._oid_1} " + details=f"DSS responded with {qcheck.response.status_code} to attempt to access OI {self._oid_1} " f"while this OI should have been available.", query_timestamps=[qcheck.request.timestamp], ) @@ -395,8 +379,7 @@ def _check_mutation_on_non_owned_intent_fails(self): if op_1_current != self._current_ref_1: check.record_failed( f"Could update operational intent using second credentials", - Severity.High, - f"Operational intent {self._oid_1} was modified by second credentials", + details=f"Operational intent {self._oid_1} was modified by second credentials", query_timestamps=[q.request.timestamp, qcheck.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py index f40ea87afb..49f0443fc6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py @@ -1,6 +1,5 @@ from typing import Optional -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance from monitoring.uss_qualifier.resources.flight_planning import ( @@ -61,14 +60,12 @@ def _validate_clear_area(self): check.record_failed( summary="Error parsing DSS response", details=str(e), - severity=Severity.High, ) self.record_query(query) if op_intents is None: check.record_failed( summary="Error querying DSS for operational intents", details="See query", - severity=Severity.High, query_timestamps=[query.request.timestamp], ) with self.check("Area is clear") as check: @@ -83,6 +80,5 @@ def _validate_clear_area(self): check.record_failed( summary=summary, details=details, - severity=Severity.High, query_timestamps=[query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_available.md b/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_available.md index 585b1c1634..20f29376f8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_available.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_available.md @@ -4,5 +4,5 @@ This step sets the USS availability to 'Available' at the DSS. See `set_uss_available` in [test_steps.py](test_steps.py). -## USS availability successfully set to 'Available' check +## 🛑 USS availability successfully set to 'Available' check **[astm.f3548.v21.DSS0100](../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_down.md b/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_down.md index 3d965a234e..420dff5806 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_down.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/set_uss_down.md @@ -4,5 +4,5 @@ This step sets the USS availability to 'Down' at the DSS. See `set_uss_down` in [test_steps.py](test_steps.py). -## USS availability successfully set to 'Down' check +## 🛑 USS availability successfully set to 'Down' check **[astm.f3548.v21.DSS0100](../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index 2b5aed717b..1290ecfb99 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -383,7 +383,6 @@ def set_uss_available( if availability_version is None: check.record_failed( summary=f"Availability of USS {uss_sub} could not be set to available", - severity=Severity.High, details=f"DSS responded code {avail_query.status_code}; error message: {avail_query.error_message}", query_timestamps=[avail_query.request.timestamp], ) @@ -416,7 +415,6 @@ def set_uss_down( if availability_version is None: check.record_failed( summary=f"Availability of USS {uss_sub} could not be set to down", - severity=Severity.High, details=f"DSS responded code {avail_query.status_code}; error message: {avail_query.error_message}", query_timestamps=[avail_query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index f31fb2652d..7c67aaec40 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -93,12 +93,19 @@ def __exit__(self, exc_type, exc_val, exc_tb): def record_failed( self, summary: str, - severity: Severity, + severity: Optional[Severity] = None, details: str = "", query_timestamps: Optional[List[datetime]] = None, additional_data: Optional[dict] = None, ) -> None: self._outcome_recorded = True + if severity is None: + if "severity" in self._documentation and self._documentation.severity: + severity = self._documentation.severity + else: + raise ValueError( + f"Severity of check '{self._documentation.name}' was not specified at failure time and is not documented in scenario documentation" + ) if ( self._stop_fast