diff --git a/monitoring/deployment_manager/actions/test/hello_world.py b/monitoring/deployment_manager/actions/test/hello_world.py index b591c5d10b..e12d0086d3 100644 --- a/monitoring/deployment_manager/actions/test/hello_world.py +++ b/monitoring/deployment_manager/actions/test/hello_world.py @@ -1,10 +1,9 @@ -import time - from monitoring.deployment_manager import deploylib import monitoring.deployment_manager.deploylib.namespaces import monitoring.deployment_manager.deploylib.systems from monitoring.deployment_manager.infrastructure import deployment_action, Context from monitoring.deployment_manager.systems.test import hello_world +from monitoring.monitorlib.delay import sleep @deployment_action("test/hello_world/deploy") @@ -52,7 +51,7 @@ def destroy(context: Context) -> None: namespace.metadata.name, context.spec.cluster.name ) ) - time.sleep(15) + sleep(15, "destruction of hello_world system may take a few seconds") deploylib.systems.delete_resources( existing_resources, namespace, context.clients, context.log diff --git a/monitoring/mock_uss/f3548v21/routes_scd.py b/monitoring/mock_uss/f3548v21/routes_scd.py index 0bb78bc04d..e8bc6b336a 100644 --- a/monitoring/mock_uss/f3548v21/routes_scd.py +++ b/monitoring/mock_uss/f3548v21/routes_scd.py @@ -1,13 +1,17 @@ +from typing import Optional + import flask from monitoring.mock_uss.f3548v21.flight_planning import op_intent_from_flightrecord from monitoring.monitorlib import scd from monitoring.mock_uss import webapp from monitoring.mock_uss.auth import requires_scope -from monitoring.mock_uss.flights.database import db +from monitoring.mock_uss.flights.database import db, FlightRecord from uas_standards.astm.f3548.v21.api import ( ErrorResponse, GetOperationalIntentDetailsResponse, + GetOperationalIntentTelemetryResponse, + OperationalIntentState, ) @@ -44,6 +48,58 @@ def scdsc_get_operational_intent_details(entityid: str): return flask.jsonify(response), 200 +@webapp.route( + "/mock/scd/uss/v1/operational_intents//telemetry", methods=["GET"] +) +@requires_scope(scd.SCOPE_CM_SA) +def scdsc_get_operational_intent_telemetry(entityid: str): + """Implements getOperationalIntentTelemetry in ASTM SCD API.""" + + # Look up entityid in database + tx = db.value + flight: Optional[FlightRecord] = None + for f in tx.flights.values(): + if f and f.op_intent.reference.id == entityid: + flight = f + break + + # If requested operational intent doesn't exist, return 404 + if flight is None: + return ( + flask.jsonify( + ErrorResponse( + message="Operational intent {} not known by this USS".format( + entityid + ) + ) + ), + 404, + ) + + elif flight.op_intent.reference.state not in { + OperationalIntentState.Contingent, + OperationalIntentState.Nonconforming, + }: + return ( + flask.jsonify( + ErrorResponse( + message=f"Operational intent {entityid} is not in a state that provides telemetry ({flight.op_intent.reference.state})" + ) + ), + 409, + ) + + # TODO: implement support for telemetry + return ( + flask.jsonify( + ErrorResponse( + message=f"Operational intent {entityid} has no telemetry data available." + ) + ), + 412, + ) + + @webapp.route("/mock/scd/uss/v1/operational_intents", methods=["POST"]) @requires_scope(scd.SCOPE_SC) def scdsc_notify_operational_intent_details_changed(): diff --git a/monitoring/mock_uss/flights/planning.py b/monitoring/mock_uss/flights/planning.py index 7acb6baa61..c2a9b4eee1 100644 --- a/monitoring/mock_uss/flights/planning.py +++ b/monitoring/mock_uss/flights/planning.py @@ -3,6 +3,7 @@ from typing import Callable, Optional from monitoring.mock_uss.flights.database import FlightRecord, db, DEADLOCK_TIMEOUT +from monitoring.monitorlib.delay import sleep def lock_flight(flight_id: str, log: Callable[[str], None]) -> FlightRecord: @@ -25,7 +26,7 @@ def lock_flight(flight_id: str, log: Callable[[str], None]) -> FlightRecord: break # We found an existing flight but it was locked; wait for it to become # available - time.sleep(0.5) + sleep(0.5, f"flight {flight_id} is currently already locked") if datetime.utcnow() > deadline: raise RuntimeError( @@ -61,7 +62,10 @@ def delete_flight_record(flight_id: str) -> Optional[FlightRecord]: # No FlightRecord found return None # There is a race condition with another handler to create or modify the requested flight; wait for that to resolve - time.sleep(0.5) + sleep( + 0.5, + f"flight {flight_id} is currently already locked while we are trying to delete it", + ) if datetime.utcnow() > deadline: raise RuntimeError( f"Deadlock in delete_flight while attempting to gain access to flight {flight_id} (now: {datetime.utcnow()}, deadline: {deadline})" diff --git a/monitoring/mock_uss/ridsp/behavior.py b/monitoring/mock_uss/ridsp/behavior.py index 6a3c3cd141..0e23723012 100644 --- a/monitoring/mock_uss/ridsp/behavior.py +++ b/monitoring/mock_uss/ridsp/behavior.py @@ -1,6 +1,6 @@ -from time import sleep from typing import Optional +from monitoring.monitorlib.delay import sleep from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight from implicitdict import ImplicitDict from uas_standards.astm.f3411.v19.api import RIDFlight @@ -56,6 +56,9 @@ def adjust_reported_flight( p.position.alt *= FEET_PER_METER if behavior.delay_flight_report_s > 0: - sleep(behavior.delay_flight_report_s) + sleep( + behavior.delay_flight_report_s, + "specified Service Provider behavior is to delay before reporting flight", + ) return adjusted diff --git a/monitoring/monitorlib/delay.py b/monitoring/monitorlib/delay.py new file mode 100644 index 0000000000..ab1df44286 --- /dev/null +++ b/monitoring/monitorlib/delay.py @@ -0,0 +1,27 @@ +from datetime import timedelta +import time +from typing import Union + +from loguru import logger + + +MAX_SILENT_DELAY_S = 0.4 +"""Number of seconds to delay above which a reasoning message should be displayed.""" + + +def sleep(duration: Union[float, timedelta], reason: str) -> None: + """Sleep for the specified amount of time, logging the fact that the delay is occurring (when appropriate). + + Args: + duration: Amount of time to sleep for; interpreted as seconds if float. + reason: Reason the delay is happening (to be printed to console/log if appropriate). + """ + if isinstance(duration, timedelta): + duration = duration.total_seconds() + if duration <= 0: + # No need to delay + return + + if duration > MAX_SILENT_DELAY_S: + logger.debug(f"Delaying {duration:.1f} seconds because {reason}") + time.sleep(duration) diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 2cddcc87c2..5f08013028 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -254,6 +254,9 @@ class QueryType(str, Enum): "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" ) + def __str__(self): + return self.value + @staticmethod def flight_details(rid_version: RIDVersion): if rid_version == RIDVersion.f3411_19: diff --git a/monitoring/prober/rid/v1/test_isa_expiry.py b/monitoring/prober/rid/v1/test_isa_expiry.py index 94d1dabd76..2518ad7999 100644 --- a/monitoring/prober/rid/v1/test_isa_expiry.py +++ b/monitoring/prober/rid/v1/test_isa_expiry.py @@ -1,8 +1,8 @@ """Test ISAs aren't returned after they expire.""" import datetime -import time +from monitoring.monitorlib.delay import sleep from monitoring.monitorlib.infrastructure import default_scope from monitoring.monitorlib import rid_v1 from monitoring.prober.infrastructure import register_resource_type @@ -64,7 +64,7 @@ def test_valid_immediately(ids, session_ridv1): def test_sleep_5_seconds(): # But if we wait 5 seconds it will expire... - time.sleep(5) + sleep(5, "if we wait 5 seconds, the ISA of interest will expire") @default_scope(Scope.Read) diff --git a/monitoring/prober/rid/v2/test_isa_expiry.py b/monitoring/prober/rid/v2/test_isa_expiry.py index d4caf15fb4..17840d0399 100644 --- a/monitoring/prober/rid/v2/test_isa_expiry.py +++ b/monitoring/prober/rid/v2/test_isa_expiry.py @@ -1,8 +1,8 @@ """Test ISAs aren't returned after they expire.""" import datetime -import time +from monitoring.monitorlib.delay import sleep from uas_standards.astm.f3411.v22a.api import OPERATIONS, OperationID from uas_standards.astm.f3411.v22a.constants import Scope @@ -68,7 +68,7 @@ def test_valid_immediately(ids, session_ridv2): def test_sleep_5_seconds(): # But if we wait 5 seconds it will expire... - time.sleep(5) + sleep(5, "if we wait 5 seconds, the ISA of interest will expire") @default_scope(Scope.DisplayProvider) diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index a03f3b758c..16f75fcd60 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -26,6 +26,7 @@ TestScenarioReport, PassedCheck, FailedCheck, + Severity, SkippedActionReport, ErrorReport, ) @@ -150,6 +151,7 @@ def rows(self) -> int: @dataclass class TestedParticipant(object): has_failures: bool = False + has_infos: bool = False has_successes: bool = False has_queries: bool = False @@ -337,7 +339,10 @@ def append_notes(new_notes): ) for pid in participants: p = scenario_participants.get(pid, TestedParticipant()) - p.has_failures = True + if failed_check.severity == Severity.Low: + p.has_infos = True + else: + p.has_failures = True scenario_participants[pid] = p if "notes" in report and report.notes: for key, note in report.notes.items(): @@ -618,6 +623,7 @@ def _generate_scenario_pages( UNATTRIBUTED_PARTICIPANT=UNATTRIBUTED_PARTICIPANT, len=len, str=str, + Severity=Severity, ) ) else: @@ -654,5 +660,6 @@ def generate_sequence_view( ActionNodeType=ActionNodeType, UNATTRIBUTED_PARTICIPANT=UNATTRIBUTED_PARTICIPANT, len=len, + Severity=Severity, ) ) diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html index 6e6f5d638e..d20a5e531f 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html @@ -3,71 +3,7 @@ TR-{{ test_run.test_run_id[0:7] }} sequence view - + {% include "sequence_view/style.html" %} {{ explorer_header() }} @@ -200,6 +136,8 @@

Scenarios executed

{% if row.scenario_node and participant_id in row.scenario_node.scenario.participants %} {% if row.scenario_node.scenario.participants[participant_id].has_failures %} ❌ + {% elif row.scenario_node.scenario.participants[participant_id].has_infos %} + ℹ️ {% elif row.scenario_node.scenario.participants[participant_id].has_successes %} ✅ {% elif row.scenario_node.scenario.participants[participant_id].has_queries %} diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html index f7f7aed65d..c6a2a03d67 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html @@ -1,69 +1,11 @@ {% from "explorer.html" import explorer_header, explorer_content, explorer_footer %} +{% from "sequence_view/severity.html" import severity_symbol, severity_class with context %} s{{ test_scenario.scenario_index }} - {{ test_scenario.name }} - + {% include "sequence_view/style.html" %} {{ explorer_header() }} @@ -131,8 +73,8 @@

{{ test_scenario.type }}

{% endif %} {% endfor %} {% elif event.type == EventType.FailedCheck %} - ❌ - + {{ severity_symbol(event.failed_check.severity) }} + {% if event.failed_check.documentation_url %} {{ event.failed_check.name }} {% else %} @@ -152,7 +94,9 @@

{{ test_scenario.type }}

{% for participant_id in all_participants %} {% if (participant_id != UNATTRIBUTED_PARTICIPANT and participant_id in event.failed_check.participants) or (participant_id == UNATTRIBUTED_PARTICIPANT and not event.failed_check.participants) %} - ❌ + + {{ severity_symbol(event.failed_check.severity) }} + {% else %} {% endif %} @@ -160,9 +104,9 @@

{{ test_scenario.type }}

{% elif event.type == EventType.Query %} 🌐 - {{ event.query.request.method }} {{ event.query.request.url_hostname }} + {% set query_dict = {event.query.request.method + " " + event.query.request.url_hostname + " " + str(event.query.response.status_code): event.query} %} {% set query_id = "e" + str(event.event_index) + "query" %} - {{ explorer_content(query_id, event.query) }} + {{ explorer_content(query_id, query_dict) }} {% set collapsible.queries = collapsible.queries + [query_id] %} {% for participant_id in all_participants %} diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/severity.html b/monitoring/uss_qualifier/reports/templates/sequence_view/severity.html new file mode 100644 index 0000000000..a755cdce35 --- /dev/null +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/severity.html @@ -0,0 +1,21 @@ +{% macro severity_symbol(s) %} + {% if s == Severity.Low %} + ℹ️ + {% elif s == Severity.Medium %} + ⚠️ + {% elif s == Severity.High %} + 🛑 + {% elif s == Severity.Critical %} + ☢️ + {% else %} + ? + {% endif %} +{% endmacro %} + +{% macro severity_class(s) %} + {% if s == Severity.Low %} + info_result + {% else %} + fail_result + {% endif %} +{% endmacro %} diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/style.html b/monitoring/uss_qualifier/reports/templates/sequence_view/style.html new file mode 100644 index 0000000000..05095c4925 --- /dev/null +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/style.html @@ -0,0 +1,72 @@ + + diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index aabb03bc50..f9889b6d12 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -7,7 +7,7 @@ from monitoring.monitorlib import infrastructure, fetch from monitoring.monitorlib.fetch import QueryType -from monitoring.monitorlib.scd import SCOPE_SC, SCOPE_AA +from monitoring.monitorlib.scd import SCOPE_SC, SCOPE_AA, SCOPE_CM_SA from monitoring.uss_qualifier.resources.resource import Resource from monitoring.uss_qualifier.resources.communications import AuthAdapterResource from uas_standards.astm.f3548.v21.api import ( @@ -30,6 +30,8 @@ GetOperationalIntentReferenceResponse, OPERATIONS, OperationID, + GetOperationalIntentTelemetryResponse, + VehicleTelemetry, ) # A base URL for a USS that is not expected to be ever called @@ -164,6 +166,29 @@ def get_full_op_intent_without_validation( return result, query + def get_op_intent_telemetry( + self, + op_intent_ref: OperationalIntentReference, + uss_participant_id: Optional[str] = None, + ) -> Tuple[Optional[VehicleTelemetry], fetch.Query]: + op = OPERATIONS[OperationID.GetOperationalIntentTelemetry] + query = fetch.query_and_describe( + self.client, + op.verb, + f"{op_intent_ref.uss_base_url}{op.path.format(entityid=op_intent_ref.id)}", + QueryType.F3548v21USSGetOperationalIntentTelemetry, + uss_participant_id, + scope=SCOPE_CM_SA, + ) + if query.status_code == 200: + result: GetOperationalIntentTelemetryResponse = ImplicitDict.parse( + query.response.json, GetOperationalIntentTelemetryResponse + ) + telemetry = result.telemetry if "telemetry" in result else None + return telemetry, query + else: + return None, query + def put_op_intent( self, extents: List[Volume4D], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py index 1007381a3e..f9e973928c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py @@ -4,6 +4,7 @@ import arrow +from monitoring.monitorlib.delay import sleep from monitoring.prober.infrastructure import register_resource_type from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource @@ -83,7 +84,7 @@ def _check_expiry_behaviors(self): ) # Wait for it to expire - time.sleep(5) + sleep(5, "we need to wait for the short-lived ISA to expire") # Search for ISAs: we should not find the expired one with self.check( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py index 93573d0a5e..fb5ded23af 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py @@ -1,6 +1,5 @@ import ipaddress import socket -import time import uuid from dataclasses import dataclass import datetime @@ -10,6 +9,7 @@ import s2sphere +from monitoring.monitorlib.delay import sleep from monitoring.monitorlib.fetch.rid import ISA from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import ( @@ -499,8 +499,10 @@ def step9(self): """Expired ISA automatically removed, ISA modifications accessible from all non-primary DSSs""" - # sleep X seconds for ISA_1 to expire - time.sleep(SHORT_WAIT_SEC) + sleep( + SHORT_WAIT_SEC, + "ISA_1 needs to expire so we can check it is automatically removed", + ) isa_1 = self._context["isa_1"] @@ -590,7 +592,10 @@ def step11(self): def step12(self): """Expired Subscriptions don’t trigger subscription notification requests""" - time.sleep(SHORT_WAIT_SEC) + sleep( + SHORT_WAIT_SEC, + "ISA needs to expire so we can check it doesn't trigger notifications", + ) isa_3 = self._new_isa("isa_3") all_sub_2_ids = self._get_entities_by_prefix("sub_2_").keys() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py b/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py index 4c7033cf21..040e71f052 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py @@ -1,4 +1,3 @@ -import time from datetime import timedelta, datetime from typing import Optional, Callable, List from loguru import logger @@ -6,6 +5,7 @@ import arrow from s2sphere import LatLngRect +from monitoring.monitorlib.delay import sleep from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( InjectedFlightCollection, ) @@ -114,7 +114,6 @@ def start_polling( break delay = t_next - arrow.utcnow() if delay.total_seconds() > 0: - logger.debug( - f"Waiting {delay.total_seconds()} seconds before polling RID system again..." + sleep( + delay, "RID sytem doesn't need to be polled again until this time" ) - time.sleep(delay.total_seconds()) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py index 20c178b88d..52cb6958d0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta import re from typing import Callable, Dict, List, Tuple, Optional -import time import arrow from implicitdict import StringBasedDateTime @@ -13,6 +12,7 @@ from monitoring.monitorlib.clients.mock_uss.interactions import Interaction from monitoring.monitorlib.clients.mock_uss.interactions import QueryDirection +from monitoring.monitorlib.delay import sleep from monitoring.monitorlib.fetch import QueryError, Query from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient @@ -54,7 +54,10 @@ def expect_mock_uss_receives_op_intent_notification( break dt = (wait_until - arrow.utcnow().datetime).total_seconds() if dt > 0: - time.sleep(min(dt, WAIT_INTERVAL_SECONDS)) + sleep( + min(dt, WAIT_INTERVAL_SECONDS), + "the expected notification was not found yet", + ) with scenario.check("Expect Notification sent", [participant_id]) as check: if not found: @@ -77,8 +80,10 @@ def expect_no_interuss_post_interactions( st: the earliest time a notification may have been sent participant_id: id of the participant responsible to send the notification """ - # Wait for next MaxTimeToWaitForSubscriptionNotificationSeconds duration to capture any notification - time.sleep(max_wait_time) + sleep( + max_wait_time, + "we have to wait the longest it may take a USS to send a notification before we can establish that they didn't send a notification", + ) found, query = mock_uss_interactions( scenario=scenario, mock_uss=mock_uss, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py index bde5bb26da..e9549c930b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/test_steps.py @@ -5,7 +5,6 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import schema_validation, fetch from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient -from monitoring.monitorlib.geotemporal import Volume4DCollection from uas_standards.astm.f3548.v21.api import ( OperationalIntentState, Volume4D, @@ -31,11 +30,6 @@ ) from uas_standards.interuss.automated_testing.scd.v1.api import InjectFlightRequest -OI_DATA_FORMAT = "Operational intent details data format" -OI_CORRECT_DETAILS = "Correct operational intent details" -OFF_NOM_VOLS = "Off-nominal volumes" -VERTICES = "Vertices" - class OpIntentValidator(object): """ @@ -165,109 +159,26 @@ def expect_shared( :returns: the shared operational intent reference. None if skipped because not found. """ - oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) - - oi_full, oi_full_query = self._dss.get_full_op_intent( - oi_ref, self._flight_planner.participant_id - ) - self._scenario.record_query(oi_full_query) - self._operational_intent_retrievable_check(oi_full_query, oi_ref.id) - - validation_failures = self._evaluate_op_intent_validation(oi_full_query) + if isinstance(flight_intent, InjectFlightRequest): + flight_intent = FlightInfo.from_scd_inject_flight_request(flight_intent) - with self._scenario.check( - OI_DATA_FORMAT, - [self._flight_planner.participant_id], - ) as check: - data_format_fail = ( - self._expected_validation_failure_found( - validation_failures, OpIntentValidationFailureType.DataFormat - ) - if validation_failures - else None - ) - if data_format_fail: - errors = data_format_fail.errors - check.record_failed( - summary="Operational intent details response failed schema validation", - severity=Severity.Medium, - details="The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" - + "\n".join( - f"At {e.json_path} in the response: {e.message}" for e in errors - ), - query_timestamps=[oi_full_query.request.timestamp], - ) - - with self._scenario.check( - OI_CORRECT_DETAILS, [self._flight_planner.participant_id] - ) as check: - priority = ( - flight_intent.operational_intent.priority - if isinstance(flight_intent, InjectFlightRequest) - else flight_intent.astm_f3548_21.priority - ) - if isinstance(flight_intent, InjectFlightRequest): - priority = flight_intent.operational_intent.priority - vols = Volume4DCollection.from_interuss_scd_api( - flight_intent.operational_intent.volumes - + flight_intent.operational_intent.off_nominal_volumes - ) - elif isinstance(flight_intent, FlightInfo): - priority = flight_intent.astm_f3548_21.priority - vols = flight_intent.basic_information.area - - error_text = validate_op_intent_details( - oi_full.details, - priority, - vols.bounding_volume.to_f3548v21(), - ) - if error_text: - check.record_failed( - summary="Operational intent details do not match user flight intent", - severity=Severity.High, - details=error_text, - query_timestamps=[oi_full_query.request.timestamp], - ) + self._begin_step() + oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) + if oi_ref is None: + self._scenario.end_test_step() + return None - with self._scenario.check( - OFF_NOM_VOLS, [self._flight_planner.participant_id] - ) as check: - off_nom_vol_fail = ( - self._expected_validation_failure_found( - validation_failures, - OpIntentValidationFailureType.NominalWithOffNominalVolumes, - ) - if validation_failures - else None - ) - if off_nom_vol_fail: - check.record_failed( - summary="Accepted or Activated operational intents are not allowed off-nominal volumes", - severity=Severity.Medium, - details=off_nom_vol_fail.error_text, - query_timestamps=[oi_full_query.request.timestamp], - ) + self._check_op_intent_details(flight_intent, oi_ref) - with self._scenario.check( - VERTICES, [self._flight_planner.participant_id] - ) as check: - vertices_fail = ( - self._expected_validation_failure_found( - validation_failures, OpIntentValidationFailureType.VertexCount - ) - if validation_failures - else None - ) - if vertices_fail: - check.record_failed( - summary="Too many vertices", - severity=Severity.Medium, - details=vertices_fail.error_text, - query_timestamps=[oi_full_query.request.timestamp], - ) + # Check telemetry if intent is off-nominal + if flight_intent.basic_information.uas_state in { + UasState.OffNominal, + UasState.Contingent, + }: + self._check_op_intent_telemetry(oi_ref) self._scenario.end_test_step() - return oi_full.reference + return oi_ref def expect_shared_with_invalid_data( self, @@ -287,8 +198,14 @@ def expect_shared_with_invalid_data( :returns: the shared operational intent reference. None if skipped because not found. """ + if isinstance(flight_intent, InjectFlightRequest): + flight_intent = FlightInfo.from_scd_inject_flight_request(flight_intent) + self._begin_step() oi_ref = self._operational_intent_shared_check(flight_intent, skip_if_not_found) + if oi_ref is None: + self._scenario.end_test_step() + return None goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( oi_ref, self._flight_planner.participant_id @@ -335,11 +252,9 @@ def _operational_intent_retrievable_check( def _operational_intent_shared_check( self, - flight_intent: Union[InjectFlightRequest | FlightInfo], + flight_intent: FlightInfo, skip_if_not_found: bool, - ) -> OperationalIntentReference: - - self._begin_step() + ) -> Optional[OperationalIntentReference]: with self._scenario.check( "Operational intent shared correctly", [self._flight_planner.participant_id] @@ -360,7 +275,6 @@ def _operational_intent_shared_check( f"{self._flight_planner.participant_id} skipped step", f"No new operational intent was found in DSS, instructed to skip test step '{self._test_step}'.", ) - self._scenario.end_test_step() return None oi_ref = self._new_oi_ref @@ -373,23 +287,11 @@ def _operational_intent_shared_check( if modified_oi_ref is None: if not skip_if_not_found: if ( - (isinstance(flight_intent, InjectFlightRequest)) - and ( - flight_intent.operational_intent.state - == OperationalIntentState.Activated - ) - ) or ( - isinstance(flight_intent, FlightInfo) - and ( - ( - flight_intent.basic_information.uas_state - == UasState.Nominal - ) - and ( - flight_intent.basic_information.usage_state - == AirspaceUsageState.InUse - ) - ) + flight_intent.basic_information.uas_state + == UasState.Nominal + ) and ( + flight_intent.basic_information.usage_state + == AirspaceUsageState.InUse ): with self._scenario.check( "Operational intent for active flight not deleted", @@ -415,7 +317,6 @@ def _operational_intent_shared_check( f"{self._flight_planner.participant_id} skipped step", f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS, instructed to skip test step '{self._test_step}'.", ) - self._scenario.end_test_step() return None oi_ref = modified_oi_ref @@ -432,6 +333,118 @@ def _operational_intent_shared_check( return oi_ref + def _check_op_intent_details( + self, flight_intent: FlightInfo, oi_ref: OperationalIntentReference + ): + oi_full, oi_full_query = self._dss.get_full_op_intent( + oi_ref, self._flight_planner.participant_id + ) + self._scenario.record_query(oi_full_query) + self._operational_intent_retrievable_check(oi_full_query, oi_ref.id) + + validation_failures = self._evaluate_op_intent_validation(oi_full_query) + with self._scenario.check( + "Operational intent details data format", + [self._flight_planner.participant_id], + ) as check: + data_format_fail = ( + self._expected_validation_failure_found( + validation_failures, OpIntentValidationFailureType.DataFormat + ) + if validation_failures + else None + ) + if data_format_fail: + errors = data_format_fail.errors + check.record_failed( + summary="Operational intent details response failed schema validation", + severity=Severity.Medium, + details="The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" + + "\n".join( + f"At {e.json_path} in the response: {e.message}" for e in errors + ), + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + "Correct operational intent details", [self._flight_planner.participant_id] + ) as check: + error_text = validate_op_intent_details( + oi_full.details, + flight_intent.astm_f3548_21.priority, + flight_intent.basic_information.area.bounding_volume.to_f3548v21(), + ) + if error_text: + check.record_failed( + summary="Operational intent details do not match user flight intent", + severity=Severity.High, + details=error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + "Off-nominal volumes", [self._flight_planner.participant_id] + ) as check: + off_nom_vol_fail = ( + self._expected_validation_failure_found( + validation_failures, + OpIntentValidationFailureType.NominalWithOffNominalVolumes, + ) + if validation_failures + else None + ) + if off_nom_vol_fail: + check.record_failed( + summary="Accepted or Activated operational intents are not allowed off-nominal volumes", + severity=Severity.Medium, + details=off_nom_vol_fail.error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + with self._scenario.check( + "Vertices", [self._flight_planner.participant_id] + ) as check: + vertices_fail = ( + self._expected_validation_failure_found( + validation_failures, OpIntentValidationFailureType.VertexCount + ) + if validation_failures + else None + ) + if vertices_fail: + check.record_failed( + summary="Too many vertices", + severity=Severity.Medium, + details=vertices_fail.error_text, + query_timestamps=[oi_full_query.request.timestamp], + ) + + def _check_op_intent_telemetry(self, oi_ref: OperationalIntentReference): + oi_tel, oi_tel_query = self._dss.get_op_intent_telemetry( + oi_ref, self._flight_planner.participant_id + ) + self._scenario.record_query(oi_tel_query) + + with self._scenario.check( + "Operational intent telemetry retrievable", + [self._flight_planner.participant_id], + ) as check: + if oi_tel_query.status_code not in {200, 412}: + check.record_failed( + summary="Operational intent telemetry could not be retrieved from USS", + severity=Severity.High, + details=f"Received status code {oi_tel_query.status_code} from {self._flight_planner.participant_id} when querying for telemetry of operational intent {oi_ref.id}", + query_timestamps=[oi_tel_query.request.timestamp], + ) + + if oi_tel is None: + check.record_failed( + summary="Warning (not a failure): USS indicated that no operational intent telemetry was available", + severity=Severity.Low, + details=f"Received status code {oi_tel_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {oi_ref.id}", + query_timestamps=[oi_tel_query.request.timestamp], + ) + def _evaluate_op_intent_validation( self, oi_full_query: fetch.Query ) -> Set[OpIntentValidationFailure]: @@ -485,13 +498,17 @@ def volume_vertices(v4): f"Operational intent {oi_full.reference.id} had too many total vertices - {n_vertices}", ) validation_failures.add( - validation_failure_type=OpIntentValidationFailureType.VertexCount, - error_text=details, + OpIntentValidationFailure( + validation_failure_type=OpIntentValidationFailureType.VertexCount, + error_text=details, + ) ) except (KeyError, ValueError) as e: validation_failures.add( - validation_failure_type=OpIntentValidationFailureType.DataFormat, - error_text=e, + OpIntentValidationFailure( + validation_failure_type=OpIntentValidationFailureType.DataFormat, + error_text=e, + ) ) return validation_failures diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md index cf89e31a83..694a381462 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md @@ -36,3 +36,9 @@ If the operational intent details reported by the USS do not match the user's fl ## ⚠️ Vertices check **[astm.f3548.v21.OPIN0020](../../../requirements/astm/f3548/v21.md)** + +## 🛑 Operational intent telemetry retrievable check + +If the operational intent is in an off-nominal state and that its telemetry cannot be retrieved from the USS, this check will fail per **[astm.f3548.v21.SCD0100](../../../requirements/astm/f3548/v21.md)**. + +The USS may explicitly indicate that no telemetry is available for this operational intent, in which case, as a warning, this check will fail with a low severity. diff --git a/monitoring/uss_qualifier/scenarios/dev/noop.py b/monitoring/uss_qualifier/scenarios/dev/noop.py index 7272905cc0..6980c7eabc 100644 --- a/monitoring/uss_qualifier/scenarios/dev/noop.py +++ b/monitoring/uss_qualifier/scenarios/dev/noop.py @@ -1,6 +1,6 @@ -import time from datetime import datetime +from monitoring.monitorlib.delay import sleep from monitoring.uss_qualifier.resources.dev import NoOpResource from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.suites.suite import ExecutionContext @@ -21,7 +21,7 @@ def run(self, context: ExecutionContext): f"Starting at {datetime.utcnow().isoformat()}Z, sleeping for {self.sleep_secs}s...", ) - time.sleep(self.sleep_secs) + sleep(self.sleep_secs, "the no-op scenario sleeps for the specified time") self.record_note("End time", f"Ending at {datetime.utcnow().isoformat()}Z.") diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index d04f902f91..282ee6c3b5 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -33,6 +33,8 @@ def _length_of_section(values, start_of_section: int) -> int: + if start_of_section + 1 >= len(values): + return 1 level = values[start_of_section].level c = start_of_section + 1 while c < len(values): diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 6b63849983..8fe439bb04 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -31,7 +31,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -156,6 +156,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 3ff13eb844..7dfe2bd883 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -18,7 +18,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -143,6 +143,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index a3c4696715..a41e0b2ebe 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -37,6 +37,7 @@ TestSuiteReport, TestSuiteActionReport, ParticipantCapabilityEvaluationReport, + Severity, SkippedActionReport, ) from monitoring.uss_qualifier.resources.definitions import ResourceID @@ -62,9 +63,14 @@ def _print_failed_check(failed_check: FailedCheck) -> None: yaml_lines = yaml.dump(json.loads(json.dumps(failed_check))).split("\n") - logger.warning( - "New failed check:\n{}", "\n".join(" " + line for line in yaml_lines) - ) + if failed_check.severity == Severity.Low: + logger.info( + "Informational finding:\n{}", "\n".join(" " + line for line in yaml_lines) + ) + else: + logger.warning( + "New failed check:\n{}", "\n".join(" " + line for line in yaml_lines) + ) class TestSuiteAction(object): diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index e94aec6801..5b924a7e21 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -19,7 +19,7 @@ Checked in - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -144,6 +144,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index b1399f5b71..6289673632 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -454,7 +454,7 @@ ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Subscription Simple
ASTM NetRID DSS: Subscription Validation
ASTM NetRID DSS: Token Validation - astm
.f3548
.v21 + astm
.f3548
.v21 DSS0005,1 Implemented ASTM F3548 flight planners preparation
ASTM F3548-21 UTM DSS Operational Intent Reference Access Control
Off-Nominal planning: down USS
Off-Nominal planning: down USS with equal priority conflicts not permitted @@ -579,6 +579,11 @@ Implemented Data Validation of GET operational intents by USS + + SCD0100 + Implemented + Data Validation of GET operational intents by USS
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Off-Nominal planning: down USS
Validation of operational intents + USS0005 Implemented diff --git a/requirements.txt b/requirements.txt index 623231def3..c22be32137 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ graphviz==0.20.1 # uss_qualifier gunicorn==20.1.0 implicitdict==2.3.0 jsonschema==4.17.3 # uss_qualifier -jwcrypto==1.4 +jwcrypto==1.5.1 kubernetes==23.3.0 # deployment_manager locust==1.2.2 # loadtest loguru==0.6.0