From 9753ad10dfa6800224aebe87cca15102b2857010 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Fri, 20 Oct 2023 19:33:49 +0200 Subject: [PATCH] [uss_qualifier/netrid] Fix #244; rename server_id to participant_id; some refactorings and fixes (#247) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [uss_qualifier] attach participant id to net0210 check, fix Issue #244 * rename server_id -> participant_id * factor out polling; refactor some parts of scenarios * add aud override in NoAuth * fix participant_id setter * fix several issues * fix small issues * add TODO --------- Co-authored-by: Mickaël Misbach Co-authored-by: Mickaël Misbach --- monitoring/monitorlib/auth.py | 26 +- .../clients/versioning/client_interuss.py | 6 +- monitoring/monitorlib/fetch/__init__.py | 16 +- monitoring/monitorlib/fetch/rid.py | 84 ++++--- monitoring/monitorlib/geo.py | 28 +++ monitoring/monitorlib/mutate/rid.py | 30 +-- monitoring/monitorlib/mutate/scd.py | 13 +- .../uss_qualifier/reports/sequence_view.py | 4 +- .../templates/sequence_view/scenario.html | 2 +- .../resources/astm/f3548/v21/dss.py | 4 +- .../resources/interuss/mock_uss/client.py | 6 +- .../resources/netrid/observers.py | 6 +- .../resources/netrid/service_providers.py | 28 ++- .../astm/netrid/common/aggregate_checks.py | 13 +- .../astm/netrid/common/dss/isa_simple.py | 4 +- .../astm/netrid/common/dss/isa_validation.py | 2 +- .../scenarios/astm/netrid/common/dss/utils.py | 14 +- .../astm/netrid/common/misbehavior.py | 226 +++++++++--------- .../astm/netrid/common/nominal_behavior.py | 201 ++-------------- .../astm/netrid/display_data_evaluator.py | 129 +++++----- .../scenarios/astm/netrid/dss_wrapper.py | 36 +-- .../scenarios/astm/netrid/injection.py | 138 ++++++++++- .../scenarios/astm/netrid/v19/misbehavior.md | 4 +- .../scenarios/astm/netrid/v22a/misbehavior.md | 2 +- .../scenarios/astm/netrid/virtual_observer.py | 51 +++- .../suites/astm/netrid/f3411_19.md | 7 +- .../monitoring/monitorlib/fetch/Query.json | 14 +- 27 files changed, 586 insertions(+), 508 deletions(-) diff --git a/monitoring/monitorlib/auth.py b/monitoring/monitorlib/auth.py index edd75c66d1..d1df97f29f 100644 --- a/monitoring/monitorlib/auth.py +++ b/monitoring/monitorlib/auth.py @@ -53,25 +53,29 @@ class NoAuth(AuthAdapter): EXPIRATION = 3600 # seconds - def __init__(self, sub: str = "uss_noauth"): + def __init__(self, sub: str = "uss_noauth", aud_override: Optional[str] = None): super().__init__() self.sub = sub + self._aud_override = aud_override # Overrides method in AuthAdapter def issue_token(self, intended_audience: str, scopes: List[str]) -> str: timestamp = int((datetime.datetime.utcnow() - _UNIX_EPOCH).total_seconds()) + claims = { + "sub": self.sub, + "client_id": self.sub, + "scope": " ".join(scopes), + "aud": intended_audience, + "nbf": timestamp - 1, + "exp": timestamp + NoAuth.EXPIRATION, + "iss": "NoAuth", + "jti": str(uuid.uuid4()), + } + if self._aud_override is not None: + claims["aud"] = self._aud_override jwt = jwcrypto.jwt.JWT( header={"typ": "JWT", "alg": "RS256"}, - claims={ - "sub": self.sub, - "client_id": self.sub, - "scope": " ".join(scopes), - "aud": intended_audience, - "nbf": timestamp - 1, - "exp": timestamp + NoAuth.EXPIRATION, - "iss": "NoAuth", - "jti": str(uuid.uuid4()), - }, + claims=claims, algs=["RS256"], ) jwt.make_signed_token(NoAuth.dummy_private_key) diff --git a/monitoring/monitorlib/clients/versioning/client_interuss.py b/monitoring/monitorlib/clients/versioning/client_interuss.py index 07fe95d9f2..8fa9f7a328 100644 --- a/monitoring/monitorlib/clients/versioning/client_interuss.py +++ b/monitoring/monitorlib/clients/versioning/client_interuss.py @@ -18,7 +18,7 @@ class InterUSSVersioningClient(VersioningClient): def __init__(self, session: UTMClientSession, participant_id: ParticipantID): super(InterUSSVersioningClient, self).__init__(participant_id) self._session = session - self._server_id = participant_id + self._participant_id = participant_id def get_version(self, version_type: Optional[str]) -> GetVersionResponse: op = api.OPERATIONS[api.OperationID.GetVersion] @@ -29,8 +29,8 @@ def get_version(self, version_type: Optional[str]) -> GetVersionResponse: "query_type": QueryType.InterUSSVersioningGetVersion, "scope": Scope.ReadSystemVersions, } - if self._server_id: - kwargs["server_id"] = self._server_id + if self._participant_id: + kwargs["participant_id"] = self._participant_id query = query_and_describe(**kwargs) if query.status_code != 200: raise VersionQueryError( diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index dd581d90e4..1661f0eb33 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -235,7 +235,7 @@ class Query(ImplicitDict): request: RequestDescription response: ResponseDescription - server_id: Optional[str] + participant_id: Optional[str] """If specified, identifier of the USS/participant hosting the server involved in this query.""" query_type: Optional[QueryType] @@ -283,7 +283,7 @@ def describe_query( resp: requests.Response, initiated_at: datetime.datetime, query_type: Optional[QueryType] = None, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> Query: query = Query( request=describe_request(resp.request, initiated_at), @@ -291,8 +291,8 @@ def describe_query( ) if query_type is not None: query.query_type = query_type - if server_id is not None: - query.server_id = server_id + if participant_id is not None: + query.participant_id = participant_id return query @@ -301,7 +301,7 @@ def query_and_describe( verb: str, url: str, query_type: Optional[QueryType] = None, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, **kwargs, ) -> Query: """Attempt to perform a query, and then describe the results of that attempt. @@ -314,7 +314,7 @@ def query_and_describe( verb: HTTP verb to perform at the specified URL. url: URL to query. query_type: If specified, the known type of query that this is. - server_id: If specified, the participant identifier of the server being queried. + participant_id: If specified, the participant identifier of the server being queried. **kwargs: Any keyword arguments that should be applied to the .request method when invoking it. Returns: @@ -350,7 +350,7 @@ def query_and_describe( client.request(verb, url, **req_kwargs), t0, query_type=query_type, - server_id=server_id, + participant_id=participant_id, ) except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}" @@ -380,7 +380,7 @@ def query_and_describe( elapsed_s=(t1 - t0).total_seconds(), reported=StringBasedDateTime(t1), ), - server_id=server_id, + participant_id=participant_id, ) if query_type is not None: result.query_type = query_type diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index 8d3867af11..e6b26c046e 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -1,15 +1,16 @@ from __future__ import annotations + import datetime from typing import Dict, List, Optional, Any, Union -from implicitdict import ImplicitDict, StringBasedDateTime import s2sphere -from uas_standards.astm.f3411 import v19, v22a import uas_standards.astm.f3411.v19.api import uas_standards.astm.f3411.v19.constants import uas_standards.astm.f3411.v22a.api import uas_standards.astm.f3411.v22a.constants import yaml +from implicitdict import ImplicitDict, StringBasedDateTime +from uas_standards.astm.f3411 import v19, v22a from uas_standards.astm.f3411.v22a.api import RIDHeight from yaml.representer import Representer @@ -123,7 +124,7 @@ def query_flights( session: UTMClientSession, area: s2sphere.LatLngRect, include_recent_positions: bool = True, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedUSSFlights: return uss_flights( self.flights_url, @@ -131,7 +132,7 @@ def query_flights( include_recent_positions, self.rid_version, session, - server_id=server_id, + participant_id=participant_id, ) @@ -639,13 +640,30 @@ def success(self) -> bool: def errors(self) -> List[str]: raise NotImplementedError("RIDQuery.errors must be overriden") - def set_server_id(self, server_id: str): + @property + def participant_id(self) -> Optional[str]: + if self.rid_version == RIDVersion.f3411_19: + if "participant_id" in self.v19_query: + return self.v19_query.participant_id + else: + return None + elif self.rid_version == RIDVersion.f3411_22a: + if "participant_id" in self.v22a_query: + return self.v22a_query.participant_id + else: + return None + else: + raise NotImplementedError( + f"Cannot retrieve participant_id using RID version {self.rid_version}" + ) + + def set_participant_id(self, participant_id: str) -> None: if self.v19_query is not None: - self.v19_query.server_id = server_id + self.v19_query.participant_id = participant_id elif self.v22a_query is not None: - self.v22a_query.server_id = server_id + self.v22a_query.participant_id = participant_id else: - raise NotImplementedError(f"Cannot set server_id") + raise NotImplementedError(f"Cannot set participant_id") class FetchedISA(RIDQuery): @@ -721,7 +739,7 @@ def isa( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedISA: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.GetIdentificationServiceArea] @@ -732,7 +750,7 @@ def isa( op.verb, url, scope=v19.constants.Scope.Read, - server_id=server_id, + participant_id=participant_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -744,7 +762,7 @@ def isa( op.verb, url, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ) ) else: @@ -856,7 +874,7 @@ def isas( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedISAs: url_time_params = "" if start_time is not None: @@ -874,7 +892,7 @@ def isas( op.verb, url, scope=v19.constants.Scope.Read, - server_id=server_id, + participant_id=participant_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -887,7 +905,7 @@ def isas( op.verb, url, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ) ) else: @@ -964,7 +982,7 @@ def uss_flights( include_recent_positions: bool, rid_version: RIDVersion, session: UTMClientSession, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedUSSFlights: if rid_version == RIDVersion.f3411_19: query = fetch.query_and_describe( @@ -983,9 +1001,9 @@ def uss_flights( else "false", }, scope=v19.constants.Scope.Read, - server_id=server_id, + query_type=QueryType.F3411v19Flights, + participant_id=participant_id, ) - query.query_type = QueryType.F3411v19Flights return FetchedUSSFlights(v19_query=query) elif rid_version == RIDVersion.f3411_22a: params = { @@ -1004,9 +1022,9 @@ def uss_flights( flights_url, params=params, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + query_type=QueryType.F3411v22aFlights, + participant_id=participant_id, ) - query.query_type = QueryType.F3411v22aFlights return FetchedUSSFlights(v22a_query=query) else: raise NotImplementedError( @@ -1091,11 +1109,11 @@ def flight_details( enhanced_details: bool, rid_version: RIDVersion, session: UTMClientSession, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedUSSFlightDetails: url = f"{flights_url}/{flight_id}/details" if rid_version == RIDVersion.f3411_19: - kwargs = {"server_id": server_id} + kwargs = {"participant_id": participant_id} if enhanced_details: kwargs["params"] = {"enhanced": "true"} kwargs["scope"] = ( @@ -1111,7 +1129,7 @@ def flight_details( "GET", url, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ) return FetchedUSSFlightDetails(v22a_query=query) else: @@ -1163,7 +1181,7 @@ def all_flights( session: UTMClientSession, dss_base_url: str = "", enhanced_details: bool = False, - server_id: Optional[str] = None, + dss_participant_id: Optional[str] = None, ) -> FetchedFlights: t = datetime.datetime.utcnow() isa_list = isas( @@ -1173,7 +1191,7 @@ def all_flights( rid_version, session, dss_base_url, - server_id=server_id, + participant_id=dss_participant_id, ) uss_flight_queries: Dict[str, FetchedUSSFlights] = {} @@ -1185,7 +1203,9 @@ def all_flights( include_recent_positions, rid_version, session, - server_id=server_id, + # Note that we have no clue at this point which participant the flights_url is for, + # this can only be determined later by comparing injected and observed flights. + participant_id=None, ) uss_flight_queries[flights_url] = flights_for_url @@ -1197,7 +1217,7 @@ def all_flights( enhanced_details, rid_version, session, - server_id=server_id, + participant_id=None, ) uss_flight_details_queries[flight.id] = details @@ -1275,7 +1295,7 @@ def subscription( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedSubscription: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.GetSubscription] @@ -1286,7 +1306,7 @@ def subscription( op.verb, url, scope=v19.constants.Scope.Read, - server_id=server_id, + participant_id=participant_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -1298,7 +1318,7 @@ def subscription( op.verb, url, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ) ) else: @@ -1382,7 +1402,7 @@ def subscriptions( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> FetchedSubscriptions: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.SearchSubscriptions] @@ -1393,7 +1413,7 @@ def subscriptions( op.verb, url, scope=v19.constants.Scope.Read, - server_id=server_id, + participant_id=participant_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -1405,7 +1425,7 @@ def subscriptions( op.verb, url, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ) ) else: diff --git a/monitoring/monitorlib/geo.py b/monitoring/monitorlib/geo.py index 5d6c138bcb..f06be05c73 100644 --- a/monitoring/monitorlib/geo.py +++ b/monitoring/monitorlib/geo.py @@ -7,12 +7,20 @@ import s2sphere import shapely.geometry from uas_standards.astm.f3548.v21 import api as f3548v21 +from uas_standards.astm.f3411.v19 import api as f3411v19 +from uas_standards.astm.f3411.v22a import api as f3411v22a +from uas_standards.interuss.automated_testing.rid.v1 import ( + injection as f3411testing_injection, +) EARTH_CIRCUMFERENCE_KM = 40075 EARTH_CIRCUMFERENCE_M = EARTH_CIRCUMFERENCE_KM * 1000 EARTH_RADIUS_M = 40075 * 1000 / (2 * math.pi) EARTH_AREA_M2 = 4 * math.pi * math.pow(EARTH_RADIUS_M, 2) +DISTANCE_TOLERANCE_M = 0.01 +COORD_TOLERANCE_DEG = 360 / EARTH_CIRCUMFERENCE_M * DISTANCE_TOLERANCE_M + class DistanceUnits(str, Enum): M = "M" @@ -31,9 +39,29 @@ class LatLngPoint(ImplicitDict): lng: float """Longitude (degrees)""" + @staticmethod + def from_f3411( + position: Union[ + f3411v19.RIDAircraftPosition, + f3411v22a.RIDAircraftPosition, + f3411testing_injection.RIDAircraftPosition, + ] + ): + return LatLngPoint( + lat=position.lat, + lng=position.lng, + ) + def as_s2sphere(self) -> s2sphere.LatLng: return s2sphere.LatLng.from_degrees(self.lat, self.lng) + def match(self, other: LatLngPoint) -> bool: + """Determine whether two points may be mistaken for each other.""" + return ( + abs(self.lat - other.lat) < COORD_TOLERANCE_DEG + and abs(self.lng - other.lng) < COORD_TOLERANCE_DEG + ) + class Radius(ImplicitDict): value: float diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index dd42e2cfc4..1661fd41ee 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -104,7 +104,7 @@ def upsert_subscription( rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, subscription_version: Optional[str] = None, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> ChangedSubscription: mutation = "create" if subscription_version is None else "update" if rid_version == RIDVersion.f3411_19: @@ -137,7 +137,7 @@ def upsert_subscription( url, json=body, scope=v19.constants.Scope.Read, - server_id=server_id, + participant_id=participant_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -165,7 +165,7 @@ def upsert_subscription( url, json=body, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ), ) else: @@ -179,7 +179,7 @@ def delete_subscription( subscription_version: str, rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> ChangedSubscription: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.DeleteSubscription] @@ -191,7 +191,7 @@ def delete_subscription( op.verb, url, scope=v19.constants.Scope.Read, - server_id=server_id, + participant_id=participant_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -204,7 +204,7 @@ def delete_subscription( op.verb, url, scope=v22a.constants.Scope.DisplayProvider, - server_id=server_id, + participant_id=participant_id, ), ) else: @@ -259,7 +259,7 @@ def notify( isa_id: str, utm_session: infrastructure.UTMClientSession, isa: Optional[ISA] = None, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> ISAChangeNotification: # Note that optional `extents` are not specified if self.rid_version == RIDVersion.f3411_19: @@ -276,7 +276,7 @@ def notify( url, json=body, scope=v19.constants.Scope.Write, - server_id=server_id, + participant_id=participant_id, ) ) elif self.rid_version == RIDVersion.f3411_22a: @@ -294,7 +294,7 @@ def notify( url, json=body, scope=v22a.constants.Scope.ServiceProvider, - server_id=server_id, + participant_id=participant_id, ) ) else: @@ -450,7 +450,7 @@ def put_isa( rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, isa_version: Optional[str] = None, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> ISAChange: mutation = "create" if isa_version is None else "update" if rid_version == RIDVersion.f3411_19: @@ -479,7 +479,7 @@ def put_isa( url, json=body, scope=v19.constants.Scope.Write, - server_id=server_id, + participant_id=participant_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -511,7 +511,7 @@ def put_isa( url, json=body, scope=v22a.constants.Scope.ServiceProvider, - server_id=server_id, + participant_id=participant_id, ), ) else: @@ -534,7 +534,7 @@ def delete_isa( isa_version: str, rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> ISAChange: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.DeleteIdentificationServiceArea] @@ -546,7 +546,7 @@ def delete_isa( op.verb, url, scope=v19.constants.Scope.Write, - server_id=server_id, + participant_id=participant_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -559,7 +559,7 @@ def delete_isa( op.verb, url, scope=v22a.constants.Scope.ServiceProvider, - server_id=server_id, + participant_id=participant_id, ), ) else: diff --git a/monitoring/monitorlib/mutate/scd.py b/monitoring/monitorlib/mutate/scd.py index 179e2b29e0..cc22914331 100644 --- a/monitoring/monitorlib/mutate/scd.py +++ b/monitoring/monitorlib/mutate/scd.py @@ -57,7 +57,7 @@ def put_subscription( min_alt_m: float = 0, max_alt_m: float = 3048, version: Optional[str] = None, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> MutatedSubscription: body = { "extents": Volume4D.from_values( @@ -76,7 +76,12 @@ def put_subscription( url += f"/{version}" result = MutatedSubscription( fetch.query_and_describe( - utm_client, "PUT", url, json=body, scope=scd.SCOPE_SC, server_id=server_id + utm_client, + "PUT", + url, + json=body, + scope=scd.SCOPE_SC, + participant_id=participant_id, ) ) result.mutation = "update" if version else "create" @@ -87,12 +92,12 @@ def delete_subscription( utm_client: infrastructure.UTMClientSession, subscription_id: str, version: str, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ) -> MutatedSubscription: url = f"/dss/v1/subscriptions/{subscription_id}/{version}" result = MutatedSubscription( fetch.query_and_describe( - utm_client, "DELETE", url, scope=scd.SCOPE_SC, server_id=server_id + utm_client, "DELETE", url, scope=scd.SCOPE_SC, participant_id=participant_id ) ) result.mutation = "delete" diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index e56ae9dd93..cba2d75efa 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -297,8 +297,8 @@ def append_notes(new_notes): events.append(Event(query=query)) all_events.append(events[-1]) participant_id = ( - query.server_id - if "server_id" in query and query.server_id + query.participant_id + if "participant_id" in query and query.participant_id else UNATTRIBUTED_PARTICIPANT ) p = scenario_participants.get(participant_id, TestedParticipant()) diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html index 06c3db54af..1b338dbb52 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html @@ -166,7 +166,7 @@

{{ test_scenario.type }}

{% set collapsible.queries = collapsible.queries + [query_id] %} {% for participant_id in all_participants %} - {% if (participant_id != UNATTRIBUTED_PARTICIPANT and participant_id == event.query.get("server_id", None)) or (participant_id == UNATTRIBUTED_PARTICIPANT and not event.query.get("server_id", None)) %} + {% if (participant_id != UNATTRIBUTED_PARTICIPANT and participant_id == event.query.get("participant_id", None)) or (participant_id == UNATTRIBUTED_PARTICIPANT and not event.query.get("participant_id", None)) %} 🌐 {% else %} diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index c0b99aac18..b416ad3121 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -66,7 +66,7 @@ def find_op_intent( url, scope=SCOPE_SC, json=req, - server_id=self.participant_id, + participant_id=self.participant_id, ) if query.status_code != 200: result = None @@ -81,7 +81,7 @@ def get_full_op_intent( ) -> Tuple[OperationalIntent, fetch.Query]: url = f"{op_intent_ref.uss_base_url}/uss/v1/operational_intents/{op_intent_ref.id}" query = fetch.query_and_describe( - self.client, "GET", url, scope=SCOPE_SC, server_id=self.participant_id + self.client, "GET", url, scope=SCOPE_SC, participant_id=self.participant_id ) if query.status_code != 200: result = None diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py b/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py index e70ab45d10..2c6fd7f534 100644 --- a/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py +++ b/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py @@ -38,7 +38,7 @@ def get_status(self) -> fetch.Query: "GET", "/scdsc/v1/status", scope=SCOPE_SCD_QUALIFIER_INJECT, - server_id=self.participant_id, + participant_id=self.participant_id, ) def get_locality(self) -> Tuple[Optional[LocalityCode], fetch.Query]: @@ -46,7 +46,7 @@ def get_locality(self) -> Tuple[Optional[LocalityCode], fetch.Query]: self.session, "GET", "/configuration/locality", - server_id=self.participant_id, + participant_id=self.participant_id, ) if query.status_code != 200: return None, query @@ -62,7 +62,7 @@ def set_locality(self, locality_code: LocalityCode) -> fetch.Query: "PUT", "/configuration/locality", scope=MOCK_USS_CONFIG_SCOPE, - server_id=self.participant_id, + participant_id=self.participant_id, json=PutLocalityRequest(locality_code=locality_code), ) diff --git a/monitoring/uss_qualifier/resources/netrid/observers.py b/monitoring/uss_qualifier/resources/netrid/observers.py index 2448df2c79..b172672225 100644 --- a/monitoring/uss_qualifier/resources/netrid/observers.py +++ b/monitoring/uss_qualifier/resources/netrid/observers.py @@ -49,7 +49,7 @@ def observe_system( # TODO replace with 'uas_standards.interuss.automated_testing.rid.v1.constants.Scope.Observe' once # the standard is updated with https://github.com/interuss/uas_standards/pull/11/files scope="dss.read.identification_service_areas", - server_id=self.participant_id, + participant_id=self.participant_id, ) try: result = ( @@ -74,10 +74,10 @@ def observe_flight_details( # TODO replace with 'uas_standards.interuss.automated_testing.rid.v1.constants.Scope.Observe' once # the standard is updated with https://github.com/interuss/uas_standards/pull/11/files scope="dss.read.identification_service_areas", - server_id=self.participant_id, + participant_id=self.participant_id, ) # Record query metadata for later use in the aggregate checks - query.server_id = self.participant_id + query.participant_id = self.participant_id query.query_type = QueryType.flight_details(rid_version) try: result = ( diff --git a/monitoring/uss_qualifier/resources/netrid/service_providers.py b/monitoring/uss_qualifier/resources/netrid/service_providers.py index 76c02a981f..cdbc5a36ea 100644 --- a/monitoring/uss_qualifier/resources/netrid/service_providers.py +++ b/monitoring/uss_qualifier/resources/netrid/service_providers.py @@ -41,39 +41,41 @@ class NetRIDServiceProvidersSpecification(ImplicitDict): class NetRIDServiceProvider(object): participant_id: str - base_url: str - client: infrastructure.UTMClientSession + injection_base_url: str + injection_client: infrastructure.UTMClientSession local_debug: bool def __init__( self, participant_id: str, - base_url: str, + injection_base_url: str, auth_adapter: infrastructure.AuthAdapter, local_debug: bool, ): self.participant_id = participant_id - self.base_url = base_url - self.client = infrastructure.UTMClientSession(base_url, auth_adapter) + self.injection_base_url = injection_base_url + self.injection_client = infrastructure.UTMClientSession( + injection_base_url, auth_adapter + ) self.local_debug = local_debug def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Query: return fetch.query_and_describe( - self.client, + self.injection_client, "PUT", url=f"/tests/{test_id}", json=request, scope=SCOPE_RID_QUALIFIER_INJECT, - server_id=self.participant_id, + participant_id=self.participant_id, ) def delete_test(self, test_id: str, version: str) -> fetch.Query: return fetch.query_and_describe( - self.client, + self.injection_client, "DELETE", url=f"/tests/{test_id}/{version}", scope=SCOPE_RID_QUALIFIER_INJECT, - server_id=self.participant_id, + participant_id=self.participant_id, ) @@ -87,10 +89,10 @@ def __init__( ): self.service_providers = [ NetRIDServiceProvider( - s.participant_id, - s.injection_base_url, - auth_adapter.adapter, - s.get("local_debug", False), + participant_id=s.participant_id, + injection_base_url=s.injection_base_url, + auth_adapter=auth_adapter.adapter, + local_debug=s.get("local_debug", False), ) for s in specification.service_providers ] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index a284f4465e..65cd73015d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -51,8 +51,9 @@ def __init__( # identify SPs and observers by their base URL self._participants_by_base_url.update( - {sp.base_url: sp.participant_id for sp in self._service_providers} + {sp.injection_base_url: sp.participant_id for sp in self._service_providers} ) + self._participants_by_base_url.update( {dp.base_url: dp.participant_id for dp in self._observers} ) @@ -85,12 +86,12 @@ def __init__( break # Only consider queries with the participant/server explicitly identified - if query.has_field_with_value("server_id"): + if query.has_field_with_value("participant_id"): participant_queries = self._queries_by_participant.get( - query.server_id, [] + query.participant_id, [] ) participant_queries.append(query) - self._queries_by_participant[query.server_id] = participant_queries + self._queries_by_participant[query.participant_id] = participant_queries def run(self): self.begin_test_scenario() @@ -101,7 +102,7 @@ def run(self): for sp in self._service_providers: self.record_note( "service_providers", - f"configured service providers: {sp.participant_id} - {sp.base_url}", + f"configured service providers: {sp.participant_id} - {sp.injection_base_url}", ) for o in self._observers: @@ -151,7 +152,7 @@ def _verify_https_everywhere(self): unattr_queries = [ query.request.url for query in self._queries - if query.get("server_id") is None + if query.get("participant_id") is None ] if len(unattr_queries) > 0: self.record_note( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index dd347acddc..c1d6dc5ae1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -76,7 +76,7 @@ def _delete_isa_if_exists(self): self._isa_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self.record_query(fetched.query) with self.check("Successful ISA query", [self._dss.participant_id]) as check: @@ -94,7 +94,7 @@ def _delete_isa_if_exists(self): fetched.isa.version, self._dss.rid_version, self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self.record_query(deleted.dss_query.query) for subscriber_id, notification in deleted.notifications.items(): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py index 3616dfec5c..8a4f355856 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py @@ -111,7 +111,7 @@ def _delete_isa_if_exists(self): isa_id=self._isa_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss_wrapper.participant_id, + participant_id=self._dss_wrapper.participant_id, ) def _isa_huge_area_check(self) -> (str, Dict[str, Any]): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py index c108b93f96..ea314377e9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py @@ -13,21 +13,21 @@ def delete_isa_if_exists( isa_id: str, rid_version: RIDVersion, session: UTMClientSession, - server_id: Optional[str] = None, + participant_id: Optional[str] = None, ): fetched = fetch.isa( isa_id, rid_version=rid_version, session=session, - server_id=server_id, + participant_id=participant_id, ) scenario.record_query(fetched.query) - with scenario.check("Successful ISA query", [server_id]) as check: + with scenario.check("Successful ISA query", [participant_id]) as check: if not fetched.success and fetched.status_code != 404: check.record_failed( "ISA information could not be retrieved", Severity.High, - f"{server_id} DSS instance returned {fetched.status_code} when queried for ISA {isa_id}", + f"{participant_id} DSS instance returned {fetched.status_code} when queried for ISA {isa_id}", query_timestamps=[fetched.query.request.timestamp], ) @@ -37,17 +37,17 @@ def delete_isa_if_exists( fetched.isa.version, rid_version, session, - server_id=server_id, + participant_id=participant_id, ) scenario.record_query(deleted.dss_query.query) for subscriber_id, notification in deleted.notifications.items(): scenario.record_query(notification.query) - with scenario.check("Removed pre-existing ISA", [server_id]) as check: + with scenario.check("Removed pre-existing ISA", [participant_id]) as check: if not deleted.dss_query.success: check.record_failed( "Could not delete pre-existing ISA", Severity.High, - f"Attempting to delete ISA {isa_id} from the {server_id} DSS returned error {deleted.dss_query.status_code}", + f"Attempting to delete ISA {isa_id} from the {participant_id} DSS returned error {deleted.dss_query.status_code}", query_timestamps=[deleted.dss_query.query.request.timestamp], ) for subscriber_url, notification in deleted.notifications.items(): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index a6fada8caf..52280c88f4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -1,23 +1,14 @@ -import time import traceback -import uuid -from typing import List +from typing import List, Set -import arrow import s2sphere -from implicitdict import ImplicitDict -from loguru import logger from requests.exceptions import RequestException -from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse +from s2sphere import LatLngRect -from monitoring.monitorlib import fetch +from monitoring.monitorlib import auth from monitoring.monitorlib.fetch import rid from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.rid import RIDVersion -from monitoring.monitorlib.rid_automated_testing.injection_api import ( - CreateTestParameters, -) -from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource from monitoring.uss_qualifier.resources.netrid import ( @@ -25,8 +16,10 @@ NetRIDServiceProviders, EvaluationConfigurationResource, ) -from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator -from monitoring.uss_qualifier.scenarios.astm.netrid.common import nominal_behavior +from monitoring.uss_qualifier.scenarios.astm.netrid import ( + injection, + display_data_evaluator, +) from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( InjectedFlightCollection, ) @@ -48,6 +41,7 @@ class Misbehavior(GenericTestScenario): _flights_data: FlightDataResource _service_providers: NetRIDServiceProviders _evaluation_configuration: EvaluationConfigurationResource + _injected_flights: List[InjectedFlight] _injected_tests: List[InjectedTest] @@ -62,8 +56,6 @@ def __init__( self._flights_data = flights_data self._service_providers = service_providers self._evaluation_configuration = evaluation_configuration - self._injected_flights = [] - self._injected_tests = [] if len(dss_pool.dss_instances) == 0: raise ValueError( "The Misbehavior Scenario requires at least one DSS instance" @@ -94,140 +86,142 @@ def run(self): self.end_test_scenario() def _inject_flights(self): - ( - self._injected_flights, - self._injected_tests, - ) = nominal_behavior.inject_flights( - test_scenario=self, - flights_data=self._flights_data, - service_providers=self._service_providers, - evaluation_configuration=self._evaluation_configuration, - realtime_period=self._rid_version.realtime_period, + (self._injected_flights, self._injected_tests) = injection.inject_flights( + self, self._flights_data, self._service_providers ) def _poll_unauthenticated_during_flights(self): config = self._evaluation_configuration.configuration + virtual_observer = VirtualObserver( + injected_flights=InjectedFlightCollection(self._injected_flights), + repeat_query_rect_period=config.repeat_query_rect_period, + min_query_diagonal_m=config.min_query_diagonal, + relevant_past_data_period=self._rid_version.realtime_period + + config.max_propagation_latency.timedelta, + ) - t_end = self._virtual_observer.get_last_time_of_interest() - t_now = arrow.utcnow() + remaining_injection_ids = set( + inj_flight.flight.injection_id for inj_flight in self._injected_flights + ) - if t_now > t_end: - raise RuntimeError( - f"Cannot evaluate RID system: injected test flights ended at {t_end}, which is before now ({t_now})" - ) + def poll_fct(rect: LatLngRect) -> bool: + nonlocal remaining_injection_ids - logger.debug(f"Polling from {t_now} until {t_end}") - for f in self._injected_flights: - span = f.flight.get_span() - logger.debug( - f"Flight {f.uss_participant_id}/{f.flight.injection_id} {span[0].isoformat()} to {span[1].isoformat()}", - ) + tested_inj_ids = self._evaluate_and_test_authentication(rect) + remaining_injection_ids -= tested_inj_ids + + # interrupt polling if there are no more injection IDs to cover + return len(remaining_injection_ids) == 0 - t_next = arrow.utcnow() - dt = config.min_polling_interval.timedelta - while arrow.utcnow() < t_end: - # Evaluate the system at an instant in time for various areas - diagonals_m = [ + virtual_observer.start_polling( + config.min_polling_interval.timedelta, + [ self._rid_version.max_diagonal_km * 1000 + 500, # too large self._rid_version.max_diagonal_km * 1000 - 100, # clustered self._rid_version.max_details_diagonal_km * 1000 - 100, # details - ] - auth_tests = [] - for diagonal_m in diagonals_m: - rect = self._virtual_observer.get_query_rect(diagonal_m) - auth_tests.append(self._evaluate_and_test_authentication(rect)) - - # If we checked for all diagonals that flights queries are properly authenticated, - # we can stop polling - if all(auth_tests): - logger.debug( - "Authentication check is complete, ending polling now.", - ) - break - - # Wait until minimum polling interval elapses - while t_next < arrow.utcnow(): - t_next += dt - if t_next > t_end: - break - delay = t_next - arrow.utcnow() - if delay.total_seconds() > 0: - logger.debug( - f"Waiting {delay.total_seconds()} seconds before polling RID system again..." - ) - time.sleep(delay.total_seconds()) + ], + poll_fct, + ) def _evaluate_and_test_authentication( self, rect: s2sphere.LatLngRect, - ) -> bool: + ) -> Set[str]: """Queries all flights in the expected way, then repeats the queries to SPs without credentials. returns true once queries to SPS have been made without credentials. False otherwise, such as when no flights were yet returned by the authenticated queries. + + :returns: set of injection IDs that were encountered and tested """ - with self.check("Missing credentials") as check: - # We grab all flights from the SP's. This is authenticated - # and is expected to succeed - sp_observation = rid.all_flights( - rect, - include_recent_positions=True, - get_details=True, - rid_version=self._rid_version, - session=self._dss.client, - server_id=self._dss.participant_id, + # We grab all flights from the SP's (which we know how to reach by first querying the DSS). + # This is authenticated and is expected to succeed + sp_observation = rid.all_flights( + rect, + include_recent_positions=True, + get_details=True, + rid_version=self._rid_version, + session=self._dss.client, + dss_participant_id=self._dss.participant_id, + ) + + mapping_by_injection_id = ( + display_data_evaluator.map_fetched_to_injected_flights( + self._injected_flights, list(sp_observation.uss_flight_queries.values()) ) - # We fish out the queries that were used to grab the flights from the SP, - # and attempt to re-query without credentials. This should fail. + ) + for q in sp_observation.queries: + self.record_query(q) + for injection_id, mapping in mapping_by_injection_id.items(): + participant_id = mapping.injected_flight.uss_participant_id + flights_url = mapping.observed_flight.query.flights_url unauthenticated_session = UTMClientSession( - prefix_url=self._dss.client.get_prefix_url(), - auth_adapter=None, - timeout_seconds=self._dss.client.timeout_seconds, + flights_url, auth.NoAuth(aud_override="") ) - queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list( - sp_observation.uss_flight_details_queries.values() + self.record_note( + f"{participant_id}/{injection_id}/missing_credentials_queries", + f"Will attempt querying with missing credentials at flights URL {flights_url} for a flights list and {len(mapping.observed_flight.query.flights)} flight details.", ) - if len(queries_to_repeat) == 0: - logger.debug("no flights queries to repeat at this point.") - return False - - logger.debug( - f"about to repeat {len(queries_to_repeat)} flights queries without credentials" - ) + with self.check("Missing credentials", [participant_id]) as check: - # Attempt to re-query the flights and flight details URLs: - for fq in queries_to_repeat: - failed_q = fetch.query_and_describe( - client=unauthenticated_session, - verb=fq.query.request.method, - url=fq.query.request.url, - json=fq.query.request.json, - data=fq.query.request.body, - server_id=self._dss.participant_id, + # check uss flights query + uss_flights_query = rid.uss_flights( + flights_url, + rect, + True, + self._rid_version, + unauthenticated_session, + participant_id, ) - logger.info( - f"Repeating query to {fq.query.request.url} without credentials" - ) - server_id = fq.query.get("server_id", "unknown") - if failed_q.response.code not in [401, 403]: + self.record_query(uss_flights_query.query) + + if uss_flights_query.success: + check.record_failed( + "Unauthenticated request for flights to USS was fulfilled", + participants=[participant_id], + severity=Severity.Medium, + details=f"Queried flights on {flights_url} for USS {participant_id} with no credentials, expected a failure but got a success reply.", + ) + elif uss_flights_query.status_code != 401: check.record_failed( - "unauthenticated request was fulfilled", - participants=[server_id], - severity=Severity.MEDIUM, - details=f"queried flights on {fq.query.request.url} with no credentials, expected a failure but got a success reply", + "Unauthenticated request for flights failed with wrong HTTP code", + participants=[participant_id], + severity=Severity.Medium, + details=f"Queried flights on {flights_url} for USS {participant_id} with no credentials, expected an HTTP 401 but got an HTTP {uss_flights_query.status_code}.", ) - else: - logger.info( - f"participant with id {server_id} properly authenticated the request" + + # check flight details query + for flight in mapping.observed_flight.query.flights: + uss_flight_details_query = rid.flight_details( + flights_url, + flight.id, + False, + self._rid_version, + unauthenticated_session, + participant_id, ) - # Keep track of the failed queries, too - self.record_query(failed_q) + self.record_query(uss_flight_details_query.query) + + if uss_flight_details_query.success: + check.record_failed( + "Unauthenticated request for flight details to USS was fulfilled", + participants=[participant_id], + severity=Severity.Medium, + details=f"Queried flight details on {flights_url} for USS {participant_id} for flight {flight.id} with no credentials, expected a failure but got a success reply.", + ) + elif uss_flight_details_query.status_code != 401: + check.record_failed( + "Unauthenticated request for flight details failed with wrong HTTP code", + participants=[participant_id], + severity=Severity.Medium, + details=f"Queried flight details on {flights_url} for USS {participant_id} for flight {flight.id} with no credentials, expected an HTTP 401 but got an HTTP {uss_flight_details_query.status_code}.", + ) - return True + return set(mapping_by_injection_id.keys()) def cleanup(self): self.begin_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index 13b34650ad..0bbca48d26 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -1,20 +1,10 @@ -import time import traceback -import uuid -from datetime import timedelta -from typing import List, Optional, Tuple +from typing import List, Optional -import arrow -from implicitdict import ImplicitDict -from loguru import logger from requests.exceptions import RequestException -from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse +from s2sphere import LatLngRect from monitoring.monitorlib.rid import RIDVersion -from monitoring.monitorlib.rid_automated_testing.injection_api import ( - CreateTestParameters, -) -from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource from monitoring.uss_qualifier.resources.netrid import ( @@ -23,7 +13,10 @@ NetRIDObserversResource, EvaluationConfigurationResource, ) -from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator +from monitoring.uss_qualifier.scenarios.astm.netrid import ( + display_data_evaluator, + injection, +) from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( InjectedFlightCollection, ) @@ -34,10 +27,7 @@ from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( VirtualObserver, ) -from monitoring.uss_qualifier.scenarios.scenario import ( - GenericTestScenario, - TestScenario, -) +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario class NominalBehavior(GenericTestScenario): @@ -62,8 +52,6 @@ def __init__( self._service_providers = service_providers self._observers = observers self._evaluation_configuration = evaluation_configuration - self._injected_flights = [] - self._injected_tests = [] self._dss_pool = dss_pool @property @@ -86,18 +74,19 @@ def run(self): self.end_test_scenario() def _inject_flights(self): - (self._injected_flights, self._injected_tests) = inject_flights( - test_scenario=self, - flights_data=self._flights_data, - service_providers=self._service_providers, - evaluation_configuration=self._evaluation_configuration, - realtime_period=self._rid_version.realtime_period, + (self._injected_flights, self._injected_tests) = injection.inject_flights( + self, self._flights_data, self._service_providers ) def _poll_during_flights(self): config = self._evaluation_configuration.configuration - - # Evaluate observed RID system states + virtual_observer = VirtualObserver( + injected_flights=InjectedFlightCollection(self._injected_flights), + repeat_query_rect_period=config.repeat_query_rect_period, + min_query_diagonal_m=config.min_query_diagonal, + relevant_past_data_period=self._rid_version.realtime_period + + config.max_propagation_latency.timedelta, + ) evaluator = display_data_evaluator.RIDObservationEvaluator( self, self._injected_flights, @@ -106,46 +95,19 @@ def _poll_during_flights(self): self._dss_pool.dss_instances[0] if self._dss_pool else None, ) - t_end = self._virtual_observer.get_last_time_of_interest() - t_now = arrow.utcnow() - if t_now > t_end: - raise RuntimeError( - f"Cannot evaluate RID system: injected test flights ended at {t_end}, which is before now ({t_now})" - ) - - logger.debug(f"Polling from {t_now} until {t_end}") - for f in self._injected_flights: - span = f.flight.get_span() - logger.debug( - f"Flight {f.uss_participant_id}/{f.flight.injection_id} {span[0].isoformat()} to {span[1].isoformat()}", - ) + def poll_fct(rect: LatLngRect) -> bool: + evaluator.evaluate_system_instantaneously(self._observers.observers, rect) + return False - t_next = arrow.utcnow() - dt = config.min_polling_interval.timedelta - while arrow.utcnow() < t_end: - # Evaluate the system at an instant in time for various areas - diagonals_m = [ + virtual_observer.start_polling( + config.min_polling_interval.timedelta, + [ self._rid_version.max_diagonal_km * 1000 + 500, # too large self._rid_version.max_diagonal_km * 1000 - 100, # clustered self._rid_version.max_details_diagonal_km * 1000 - 100, # details - ] - for diagonal_m in diagonals_m: - rect = self._virtual_observer.get_query_rect(diagonal_m) - evaluator.evaluate_system_instantaneously( - self._observers.observers, rect - ) - - # Wait until minimum polling interval elapses - while t_next < arrow.utcnow(): - t_next += dt - if t_next > t_end: - break - delay = t_next - arrow.utcnow() - if delay.total_seconds() > 0: - logger.debug( - f"Waiting {delay.total_seconds()} seconds before polling RID system again..." - ) - time.sleep(delay.total_seconds()) + ], + poll_fct, + ) def cleanup(self): self.begin_cleanup() @@ -181,116 +143,3 @@ def cleanup(self): details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}", ) self.end_cleanup() - - -def inject_flights( - test_scenario: TestScenario, - flights_data: FlightDataResource, - service_providers: NetRIDServiceProviders, - evaluation_configuration: EvaluationConfigurationResource, - realtime_period: timedelta, -) -> Tuple[List[InjectedFlight], List[InjectedTest]]: - test_id = str(uuid.uuid4()) - test_flights = flights_data.get_test_flights() - service_providers = service_providers.service_providers - - injected_flights: List[InjectedFlight] = [] - injected_tests: List[InjectedTest] = [] - - if len(service_providers) > len(test_flights): - raise ValueError( - "{} service providers were specified, but data for only {} test flights were provided".format( - len(service_providers), len(test_flights) - ) - ) - for i, target in enumerate(service_providers): - p = CreateTestParameters(requested_flights=[test_flights[i]]) - check = test_scenario.check("Successful injection", [target.participant_id]) - try: - query = target.submit_test(p, test_id) - except RequestException as e: - stacktrace = "".join( - traceback.format_exception(type(e), value=e, tb=e.__traceback__) - ) - check.record_failed( - summary="Error while trying to inject test flight", - severity=Severity.High, - details=f"While trying to inject a test flight into {target.participant_id}, encountered error:\n{stacktrace}", - ) - raise RuntimeError("High-severity issue did not abort test scenario") - test_scenario.record_query(query) - try: - if query.status_code != 200: - raise ValueError( - f"Expected response code 200 but received {query.status_code} instead" - ) - if "json" not in query.response: - raise ValueError("Response did not contain a JSON body") - changed_test: ChangeTestResponse = ImplicitDict.parse( - query.response.json, ChangeTestResponse - ) - injected_tests.append( - InjectedTest( - participant_id=target.participant_id, - test_id=test_id, - version=changed_test.version, - ) - ) - injections = changed_test.injected_flights - check.record_passed() - except ValueError as e: - check.record_failed( - summary="Error injecting test flight", - severity=Severity.High, - details=f"Attempting to inject a test flight into {target.participant_id}, encountered status code {query.status_code}: {str(e)}", - query_timestamps=[query.request.timestamp], - ) - raise RuntimeError("High-severity issue did not abort test scenario") - - start_time = None - end_time = None - for flight in injections: - injected_flights.append( - InjectedFlight( - uss_participant_id=target.participant_id, - test_id=test_id, - flight=TestFlight(flight), - query_timestamp=query.request.timestamp, - ) - ) - earliest_time = min(t.timestamp.datetime for t in flight.telemetry) - latest_time = max(t.timestamp.datetime for t in flight.telemetry) - if start_time is None or earliest_time < start_time: - start_time = earliest_time - if end_time is None or latest_time > end_time: - end_time = latest_time - now = arrow.utcnow().datetime - dt0 = (start_time - now).total_seconds() - dt1 = (end_time - now).total_seconds() - test_scenario.record_note( - f"{test_id} time range", - f"Injected flights start {dt0:.1f} seconds from now and end {dt1:.1f} seconds from now", - ) - - # Make sure the injected flights can be identified correctly by the test harness - with test_scenario.check("Identifiable flights") as check: - errors = display_data_evaluator.injected_flights_errors(injected_flights) - if errors: - check.record_failed( - "Injected flights not suitable for test", - Severity.High, - details="When checking the suitability of the flights (as injected) for the test, found:\n" - + "\n".join(errors), - query_timestamps=[f.query_timestamp for f in injected_flights], - ) - raise RuntimeError("High-severity issue did not abort test scenario") - - config = evaluation_configuration.configuration - test_scenario._virtual_observer = VirtualObserver( - injected_flights=InjectedFlightCollection(injected_flights), - repeat_query_rect_period=config.repeat_query_rect_period, - min_query_diagonal_m=config.min_query_diagonal, - relevant_past_data_period=realtime_period - + config.max_propagation_latency.timedelta, - ) - return injected_flights, injected_tests diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index e371b0fd48..956da71fda 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -6,11 +6,12 @@ import math import s2sphere from s2sphere import LatLng, LatLngRect + from monitoring.uss_qualifier.scenarios.astm.netrid.common_dictionary_evaluator import ( RIDCommonDictionaryEvaluator, ) -from monitoring.monitorlib.fetch import Query, QueryType +from monitoring.monitorlib.fetch import Query from monitoring.monitorlib.fetch.rid import ( all_flights, FetchedFlights, @@ -18,7 +19,6 @@ Position, ) from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance -from uas_standards.interuss.automated_testing.rid.v1.injection import RIDAircraftState from uas_standards.interuss.automated_testing.rid.v1.observation import ( Flight, GetDisplayDataResponse, @@ -36,15 +36,9 @@ from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( VirtualObserver, ) -from monitoring.uss_qualifier.scenarios.scenario import ( - TestScenarioType, - TestScenario, -) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.scenarios.astm.netrid.injection import InjectedFlight -DISTANCE_TOLERANCE_M = 0.01 -COORD_TOLERANCE_DEG = 360 / geo.EARTH_CIRCUMFERENCE_M * DISTANCE_TOLERANCE_M - def _rect_str(rect) -> str: return "({}, {})-({}, {})".format( @@ -55,41 +49,6 @@ def _rect_str(rect) -> str: ) -def _telemetry_match(t1: RIDAircraftState, t2: RIDAircraftState) -> bool: - """Determine whether two telemetry points may be mistaken for each other.""" - return ( - abs(t1.position.lat - t2.position.lat) < COORD_TOLERANCE_DEG - and abs(t1.position.lng - t2.position.lng) < COORD_TOLERANCE_DEG - ) - - -def injected_flights_errors(injected_flights: List[InjectedFlight]) -> List[str]: - """Determine whether each telemetry in each injected flight can be easily distinguished from each other. - - Args: - injected_flights: Full set of flights injected into Service Providers. - - Returns: List of error messages, or an empty list if no errors. - """ - errors: List[str] = [] - for f1, injected_flight in enumerate(injected_flights): - for t1, injected_telemetry in enumerate(injected_flight.flight.telemetry): - for t2, other_telemetry in enumerate( - injected_flight.flight.telemetry[t1 + 1 :] - ): - if _telemetry_match(injected_telemetry, other_telemetry): - errors.append( - f"{injected_flight.uss_participant_id}'s flight with injection ID {injected_flight.flight.injection_id} in test {injected_flight.test_id} has telemetry at indices {t1} and {t1 + 1 + t2} which can be mistaken for each other; (lat={injected_telemetry.position.lat}, lng={injected_telemetry.position.lng}) and (lat={other_telemetry.position.lat}, lng={other_telemetry.position.lng}) respectively" - ) - for f2, other_flight in enumerate(injected_flights[f1 + 1 :]): - for t2, other_telemetry in enumerate(other_flight.flight.telemetry): - if _telemetry_match(injected_telemetry, other_telemetry): - errors.append( - f"{injected_flight.uss_participant_id}'s flight with injection ID {injected_flight.flight.injection_id} in test {injected_flight.test_id} has telemetry at index {t1} that can be mistaken for telemetry index {t2} in {other_flight.uss_participant_id}'s flight with injection ID {other_flight.flight.injection_id} in test {other_flight.test_id}; (lat={injected_telemetry.position.lat}, lng={injected_telemetry.position.lng}) and (lat={other_telemetry.position.lat}, lng={other_telemetry.position.lng}) respectively" - ) - return errors - - @dataclass class DPObservedFlight(object): query: FetchedUSSFlights @@ -114,7 +73,7 @@ class TelemetryMapping(object): observed_flight: ObservationType -def _make_flight_mapping( +def map_observations_to_injected_flights( injected_flights: List[InjectedFlight], observed_flights: List[ObservationType], ) -> Dict[str, TelemetryMapping]: @@ -128,7 +87,7 @@ def _make_flight_mapping( injected_flights: Flights injected into RID Service Providers under test. observed_flights: Flight observed from an RID Display Provider under test. - Returns: Mapping betweenInjectedFlight and observed Flight, indexed by injection_id. + Returns: Mapping between InjectedFlight and observed Flight, indexed by injection_id. """ mapping: Dict[str, TelemetryMapping] = {} for injected_flight in injected_flights: @@ -154,7 +113,7 @@ def _make_flight_mapping( for t1, injected_telemetry in enumerate(injected_flight.flight.telemetry): dlat = abs(p.lat - injected_telemetry.position.lat) dlng = abs(p.lng - injected_telemetry.position.lng) - if dlat < COORD_TOLERANCE_DEG and dlng < COORD_TOLERANCE_DEG: + if dlat < geo.COORD_TOLERANCE_DEG and dlng < geo.COORD_TOLERANCE_DEG: new_distance = math.sqrt(math.pow(dlat, 2) + math.pow(dlng, 2)) if new_distance < smallest_distance: best_match = TelemetryMapping( @@ -181,6 +140,38 @@ def _make_flight_mapping( return mapping +def map_fetched_to_injected_flights( + injected_flights: List[InjectedFlight], + fetched_flights: List[FetchedUSSFlights], +) -> Dict[str, TelemetryMapping]: + """Identify which of the fetched flights (if any) matches to each of the injected flights. + + See `map_observations_to_injected_flights`. + If it is not already set, sets the observation flight's server ID to the one of the matching injected flight. + + :param injected_flights: Flights injected into RID Service Providers under test. + :param fetched_flights: Flight observed from an RID Display Provider under test. + :return: Mapping between InjectedFlight and observed Flight, indexed by injection_id. + """ + observed_flights = [] + for uss_query in fetched_flights: + for f in range(len(uss_query.flights)): + observed_flights.append(DPObservedFlight(query=uss_query, flight=f)) + + tel_mapping = map_observations_to_injected_flights( + injected_flights, observed_flights + ) + + # TODO: a better approach here would be to separately map flights URL to participant IDs based on all TelemetryMapping encountered, and set retroactively the participant ID on all queries + for mapping in tel_mapping.values(): + if mapping.observed_flight.query.participant_id is None: + mapping.observed_flight.query.set_participant_id( + mapping.injected_flight.uss_participant_id + ) + + return tel_mapping + + class RIDObservationEvaluator(object): """Evaluates observations of an RID system over time. @@ -239,12 +230,18 @@ def evaluate_system_instantaneously( get_details=True, rid_version=self._rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + dss_participant_id=self._dss.participant_id, + ) + + # map observed flights to injected flight and attribute participant ID + mapping_by_injection_id = map_fetched_to_injected_flights( + self._injected_flights, list(sp_observation.uss_flight_queries.values()) ) for q in sp_observation.queries: self._test_scenario.record_query(q) - self._evaluate_sp_observation(sp_observation, rect) + # Evaluate observations + self._evaluate_sp_observation(rect, sp_observation, mapping_by_injection_id) step_report = self._test_scenario.end_test_step() perform_observation = step_report.successful() @@ -341,7 +338,7 @@ def _evaluate_normal_observation( query_timestamps=[query.request.timestamp], ) - mapping_by_injection_id = _make_flight_mapping( + mapping_by_injection_id = map_observations_to_injected_flights( self._injected_flights, observation.flights ) @@ -381,7 +378,7 @@ def _evaluate_normal_observation( ) as check: if ( abs(observed_position.alt - injected_position.alt) - > DISTANCE_TOLERANCE_M + > geo.DISTANCE_TOLERANCE_M ): check.record_failed( "Observed altitude does not match injected altitude", @@ -782,7 +779,7 @@ def _evaluate_obfuscated_clusters_observation( * geo.EARTH_CIRCUMFERENCE_M / 360 ) - if distance <= DISTANCE_TOLERANCE_M: + if distance <= geo.DISTANCE_TOLERANCE_M: # Flight was not obfuscated check.record_failed( summary="Error while evaluating obfuscation of individual flights. Flight was not obfuscated: it is at the center of the cluster.", @@ -815,8 +812,9 @@ def _evaluate_obfuscated_clusters_observation( def _evaluate_sp_observation( self, + requested_area: s2sphere.LatLngRect, sp_observation: FetchedFlights, - rect: s2sphere.LatLngRect, + mappings: Dict[str, TelemetryMapping], ) -> None: # Note: This step currently uses the DSS endpoint to perform a one-time query for ISAs, but this # endpoint is not strictly required. The PUT Subscription endpoint, followed immediately by the @@ -839,31 +837,18 @@ def _evaluate_sp_observation( ) return - observed_flights = [] - for uss_query in sp_observation.uss_flight_queries.values(): - for f in range(len(uss_query.flights)): - observed_flights.append(DPObservedFlight(query=uss_query, flight=f)) - mapping_by_injection_id = _make_flight_mapping( - self._injected_flights, observed_flights - ) - - for telemetry_mapping in mapping_by_injection_id.values(): - # For flights that were mapped to an injection ID, - # update the observation queries with the participant id for future use in the aggregate checks - telemetry_mapping.observed_flight.query.set_server_id( - telemetry_mapping.injected_flight.uss_participant_id - ) - diagonal_km = ( - rect.lo().get_distance(rect.hi()).degrees * geo.EARTH_CIRCUMFERENCE_KM / 360 + requested_area.lo().get_distance(requested_area.hi()).degrees + * geo.EARTH_CIRCUMFERENCE_KM + / 360 ) if diagonal_km > self._rid_version.max_diagonal_km: self._evaluate_area_too_large_sp_observation( - mapping_by_injection_id, rect, diagonal_km + mappings, requested_area, diagonal_km ) else: self._evaluate_normal_sp_observation( - rect, sp_observation, mapping_by_injection_id + requested_area, sp_observation, mappings ) def _evaluate_normal_sp_observation( @@ -932,7 +917,7 @@ def _evaluate_normal_sp_observation( ) as check: if ( abs(observed_position.alt - injected_position.alt) - > DISTANCE_TOLERANCE_M + > geo.DISTANCE_TOLERANCE_M ): check.record_failed( "Altitude reported by Service Provider does not match injected altitude", diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 6c5dd91570..e49500a326 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -126,7 +126,7 @@ def search_isas( end_time=end_time, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( main_check, @@ -176,7 +176,7 @@ def search_isas_expect_response_code( end_time=end_time, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -206,7 +206,7 @@ def get_isa( isa_id=isa_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result(check, isa, f"Failed to get ISA {isa_id}") @@ -245,7 +245,7 @@ def get_isa_expect_response_code( isa_id=isa_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -282,7 +282,7 @@ def put_isa_expect_response_code( isa_version=isa_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -326,7 +326,7 @@ def put_isa( isa_version=isa_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( main_check, mutated_isa.dss_query, f"Failed to insert ISA {isa_id}" @@ -457,7 +457,7 @@ def del_isa( isa_version=isa_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( main_check, del_isa.dss_query, f"Failed to delete ISA {isa_id}" @@ -542,7 +542,7 @@ def del_isa_expect_response_code( isa_version=isa_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -570,7 +570,7 @@ def cleanup_isa( isa_id=isa_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -585,7 +585,7 @@ def cleanup_isa( isa_version=isa.isa.version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -620,7 +620,7 @@ def search_subs( area=area, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -651,7 +651,7 @@ def get_sub( subscription_id=sub_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -691,7 +691,7 @@ def no_sub( subscription_id=sub_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -734,7 +734,7 @@ def put_sub_expect_response_code( subscription_version=sub_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -783,7 +783,7 @@ def put_sub( subscription_version=sub_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -815,7 +815,7 @@ def del_sub( subscription_version=sub_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -854,7 +854,7 @@ def cleanup_sub( subscription_id=sub_id, rid_version=self._dss.rid_version, session=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( @@ -873,7 +873,7 @@ def cleanup_sub( subscription_version=sub.subscription.version, rid_version=self._dss.rid_version, utm_client=self._dss.client, - server_id=self._dss.participant_id, + participant_id=self._dss.participant_id, ) self._handle_query_result( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py b/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py index 618613b581..96452d4c26 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py @@ -1,8 +1,22 @@ +import uuid from datetime import datetime +from typing import List, Tuple +import arrow from implicitdict import ImplicitDict +from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse -from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight +from monitoring.monitorlib import geo +from monitoring.monitorlib.rid_automated_testing.injection_api import ( + TestFlight, + CreateTestParameters, +) +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.netrid import ( + FlightDataResource, + NetRIDServiceProviders, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario class InjectedFlight(ImplicitDict): @@ -16,3 +30,125 @@ class InjectedTest(ImplicitDict): participant_id: str test_id: str version: str + + +def inject_flights( + test_scenario: TestScenario, + flights_data_res: FlightDataResource, + service_providers_res: NetRIDServiceProviders, +) -> Tuple[List[InjectedFlight], List[InjectedTest]]: + test_id = str(uuid.uuid4()) + test_flights = flights_data_res.get_test_flights() + service_providers = service_providers_res.service_providers + + injected_flights: List[InjectedFlight] = [] + injected_tests: List[InjectedTest] = [] + + if len(service_providers) > len(test_flights): + raise ValueError( + f"{len(service_providers)} service providers were specified, but data for only {len(test_flights)} test flights were provided" + ) + for i, target in enumerate(service_providers): + p = CreateTestParameters(requested_flights=[test_flights[i]]) + with test_scenario.check( + "Successful injection", [target.participant_id] + ) as check: + query = target.submit_test(p, test_id) + test_scenario.record_query(query) + + if query.status_code != 200: + check.record_failed( + summary="Error while trying to inject test flight", + severity=Severity.High, + details=f"Expected response code 200 from {target.participant_id} but received {query.status_code} while trying to inject a test flight", + query_timestamps=[query.request.timestamp], + ) + + if "json" not in query.response or query.response.json is None: + check.record_failed( + summary="Response to test flight injection request did not contain a JSON body", + severity=Severity.High, + details=f"Expected a JSON body in response to flight injection request", + query_timestamps=[query.request.timestamp], + ) + + changed_test: ChangeTestResponse = ImplicitDict.parse( + query.response.json, ChangeTestResponse + ) + injected_tests.append( + InjectedTest( + participant_id=target.participant_id, + test_id=test_id, + version=changed_test.version, + ) + ) + + start_time = None + end_time = None + for flight in changed_test.injected_flights: + injected_flights.append( + InjectedFlight( + uss_participant_id=target.participant_id, + test_id=test_id, + flight=TestFlight(flight), + query_timestamp=query.request.timestamp, + ) + ) + earliest_time = min(t.timestamp.datetime for t in flight.telemetry) + latest_time = max(t.timestamp.datetime for t in flight.telemetry) + if start_time is None or earliest_time < start_time: + start_time = earliest_time + if end_time is None or latest_time > end_time: + end_time = latest_time + now = arrow.utcnow().datetime + dt0 = (start_time - now).total_seconds() + dt1 = (end_time - now).total_seconds() + test_scenario.record_note( + f"{test_id} time range", + f"Injected flights start {dt0:.1f} seconds from now and end {dt1:.1f} seconds from now", + ) + + # Make sure the injected flights can be identified correctly by the test harness + with test_scenario.check("Identifiable flights") as check: + errors = injected_flights_errors(injected_flights) + if errors: + check.record_failed( + summary="Injected flights not suitable for test", + severity=Severity.High, + details="When checking the suitability of the flights (as injected) for the test, found:\n" + + "\n".join(errors), + query_timestamps=[f.query_timestamp for f in injected_flights], + ) + + return injected_flights, injected_tests + + +def injected_flights_errors(injected_flights: List[InjectedFlight]) -> List[str]: + """Determine whether each telemetry in each injected flight can be easily distinguished from each other. + + Args: + injected_flights: Full set of flights injected into Service Providers. + + Returns: List of error messages, or an empty list if no errors. + """ + errors: List[str] = [] + for f1, injected_flight in enumerate(injected_flights): + for t1, injected_telemetry in enumerate(injected_flight.flight.telemetry): + for t2, other_telemetry in enumerate( + injected_flight.flight.telemetry[t1 + 1 :] + ): + if geo.LatLngPoint.from_f3411(injected_telemetry.position).match( + geo.LatLngPoint.from_f3411(other_telemetry.position) + ): + errors.append( + f"{injected_flight.uss_participant_id}'s flight with injection ID {injected_flight.flight.injection_id} in test {injected_flight.test_id} has telemetry at indices {t1} and {t1 + 1 + t2} which can be mistaken for each other; (lat={injected_telemetry.position.lat}, lng={injected_telemetry.position.lng}) and (lat={other_telemetry.position.lat}, lng={other_telemetry.position.lng}) respectively" + ) + for f2, other_flight in enumerate(injected_flights[f1 + 1 :]): + for t2, other_telemetry in enumerate(other_flight.flight.telemetry): + if geo.LatLngPoint.from_f3411(injected_telemetry.position).match( + geo.LatLngPoint.from_f3411(other_telemetry.position) + ): + errors.append( + f"{injected_flight.uss_participant_id}'s flight with injection ID {injected_flight.flight.injection_id} in test {injected_flight.test_id} has telemetry at index {t1} that can be mistaken for telemetry index {t2} in {other_flight.uss_participant_id}'s flight with injection ID {other_flight.flight.injection_id} in test {other_flight.test_id}; (lat={injected_telemetry.position.lat}, lng={injected_telemetry.position.lng}) and (lat={other_telemetry.position.lat}, lng={other_telemetry.position.lng}) respectively" + ) + return errors diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md index 0602647f31..af43f8b57f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -44,11 +44,11 @@ This particular test requires each flight to be uniquely identifiable by its 2D In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials to confirm that the requested data is indeed available to any authorized query. -It then repeats the exact same request while omitting the credentials, and expects this to fail. +It then repeats the exact same request with incorrect credentials, and expects this to fail. #### Missing credentials check -This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)**, +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**, and that requests for existing flights that are executed with missing or incorrect credentials fail. ## Cleanup diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md index 5089d9d23c..698084d61d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -44,7 +44,7 @@ This particular test requires each flight to be uniquely identifiable by its 2D In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials to confirm that the requested data is indeed available to any authorized query. -It then repeats the exact same request while omitting the credentials, and expects this to fail. +It then repeats the exact same request with incorrect credentials, and expects this to fail. #### Missing credentials check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py b/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py index fad4ca7759..4c7033cf21 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/virtual_observer.py @@ -1,5 +1,7 @@ +import time from datetime import timedelta, datetime -from typing import Optional +from typing import Optional, Callable, List +from loguru import logger import arrow from s2sphere import LatLngRect @@ -69,3 +71,50 @@ def get_last_time_of_interest(self) -> datetime: self._injected_flights.get_end_of_injected_data() + self._relevant_past_data_period ) + + def start_polling( + self, + interval: timedelta, + diagonals_m: List[float], + poll_fct: Callable[[LatLngRect], bool], + ) -> None: + """ + Start polling of the RID system. + + :param interval: polling interval. + :param diagonals_m: list of the query rectangle diagonals (in meters). + :param poll_fct: polling function to invoke. If it returns True, the polling will be immediately interrupted before the end. + """ + t_end = self.get_last_time_of_interest() + t_now = arrow.utcnow() + if t_now > t_end: + raise ValueError( + f"Cannot poll RID system: instructed to poll until {t_end}, which is before now ({t_now})" + ) + + logger.info(f"Polling from {t_now} until {t_end} every {interval}") + t_next = arrow.utcnow() + while arrow.utcnow() < t_end: + interrupt_polling = False + for diagonal_m in diagonals_m: + rect = self.get_query_rect(diagonal_m) + interrupt_polling = poll_fct(rect) + if interrupt_polling: + break + + if interrupt_polling: + logger.info(f"Polling ended early at {arrow.utcnow()}.") + break + + # Wait until minimum polling interval elapses + while t_next < arrow.utcnow(): + t_next += interval + if t_next > t_end: + logger.info(f"Polling ended normally at {t_end}.") + break + delay = t_next - arrow.utcnow() + if delay.total_seconds() > 0: + logger.debug( + f"Waiting {delay.total_seconds()} seconds before polling RID system again..." + ) + time.sleep(delay.total_seconds()) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 464bbd5f6f..347a1f1add 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 DSS0030,a Implemented ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA @@ -206,6 +206,11 @@ TODO ASTM NetRID: Operator interactions + + NET0210 + Implemented + ASTM NetRID SP clients misbehavior handling + NET0220 Implemented diff --git a/schemas/monitoring/monitorlib/fetch/Query.json b/schemas/monitoring/monitorlib/fetch/Query.json index 88c3188ed5..0255097971 100644 --- a/schemas/monitoring/monitorlib/fetch/Query.json +++ b/schemas/monitoring/monitorlib/fetch/Query.json @@ -7,6 +7,13 @@ "description": "Path to content that replaces the $ref", "type": "string" }, + "participant_id": { + "description": "If specified, identifier of the USS/participant hosting the server involved in this query.", + "type": [ + "string", + "null" + ] + }, "query_type": { "description": "If specified, the recognized type of this query.", "enum": [ @@ -50,13 +57,6 @@ }, "response": { "$ref": "ResponseDescription.json" - }, - "server_id": { - "description": "If specified, identifier of the USS/participant hosting the server involved in this query.", - "type": [ - "string", - "null" - ] } }, "required": [