diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 973a7dd0a3..d1db2f7c82 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -258,14 +258,14 @@ def describe_query( resp: requests.Response, initiated_at: datetime.datetime, query_type: Optional[QueryType] = None, + server_id: Optional[str] = None, ) -> Query: - result = Query( + return Query( request=describe_request(resp.request, initiated_at), response=describe_response(resp), + server_id=server_id, + query_type=query_type, ) - if query_type is not None: - result.query_type = query_type - return result def query_and_describe( @@ -296,6 +296,7 @@ def query_and_describe( else: utm_session = True req_kwargs = kwargs.copy() + req_kwargs.pop("server_id", None) if "timeout" not in req_kwargs: req_kwargs["timeout"] = TIMEOUTS @@ -307,7 +308,10 @@ def query_and_describe( t0 = datetime.datetime.utcnow() try: return describe_query( - client.request(verb, url, **req_kwargs), t0, query_type + client.request(verb, url, **req_kwargs), + t0, + query_type=query_type, + server_id=kwargs.get("server_id", None), ) except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}" @@ -337,6 +341,7 @@ def query_and_describe( elapsed_s=(t1 - t0).total_seconds(), reported=StringBasedDateTime(t1), ), + server_id=kwargs.get("server_id", None), ) if query_type is not None: result.query_type = query_type diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index 3da7beb610..bb96bf2d24 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -122,9 +122,15 @@ def query_flights( session: UTMClientSession, area: s2sphere.LatLngRect, include_recent_positions: bool = True, + server_id: Optional[str] = None, ) -> FetchedUSSFlights: return uss_flights( - self.flights_url, area, include_recent_positions, self.rid_version, session + self.flights_url, + area, + include_recent_positions, + self.rid_version, + session, + server_id=server_id, ) @@ -534,13 +540,18 @@ def isa( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", + server_id: Optional[str] = None, ) -> FetchedISA: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.GetIdentificationServiceArea] url = f"{dss_base_url}{op.path}".replace("{id}", isa_id) return FetchedISA( v19_query=fetch.query_and_describe( - session, op.verb, url, scope=v19.constants.Scope.Read + session, + op.verb, + url, + scope=v19.constants.Scope.Read, + server_id=server_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -548,7 +559,11 @@ def isa( url = f"{dss_base_url}{op.path}".replace("{id}", isa_id) return FetchedISA( v22a_query=fetch.query_and_describe( - session, op.verb, url, scope=v22a.constants.Scope.DisplayProvider + session, + op.verb, + url, + scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ) ) else: @@ -660,6 +675,7 @@ def isas( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", + server_id: Optional[str] = None, ) -> FetchedISAs: t0 = rid_version.format_time(start_time) t1 = rid_version.format_time(end_time) @@ -669,7 +685,11 @@ def isas( url = f"{dss_base_url}{op.path}?area={area}&earliest_time={t0}&latest_time={t1}" return FetchedISAs( v19_query=fetch.query_and_describe( - session, op.verb, url, scope=v19.constants.Scope.Read + session, + op.verb, + url, + scope=v19.constants.Scope.Read, + server_id=server_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -678,7 +698,11 @@ def isas( url = f"{dss_base_url}{op.path}?area={area}&earliest_time={t0}&latest_time={t1}" return FetchedISAs( v22a_query=fetch.query_and_describe( - session, op.verb, url, scope=v22a.constants.Scope.DisplayProvider + session, + op.verb, + url, + scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ) ) else: @@ -755,6 +779,7 @@ def uss_flights( include_recent_positions: bool, rid_version: RIDVersion, session: UTMClientSession, + server_id: Optional[str] = None, ) -> FetchedUSSFlights: if rid_version == RIDVersion.f3411_19: query = fetch.query_and_describe( @@ -773,6 +798,7 @@ def uss_flights( else "false", }, scope=v19.constants.Scope.Read, + server_id=server_id, ) query.query_type = QueryType.F3411v19Flights return FetchedUSSFlights(v19_query=query) @@ -793,6 +819,7 @@ def uss_flights( flights_url, params=params, scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ) query.query_type = QueryType.F3411v22aFlights return FetchedUSSFlights(v22a_query=query) @@ -879,10 +906,11 @@ def flight_details( enhanced_details: bool, rid_version: RIDVersion, session: UTMClientSession, + server_id: Optional[str] = None, ) -> FetchedUSSFlightDetails: url = f"{flights_url}/{flight_id}/details" if rid_version == RIDVersion.f3411_19: - kwargs = {} + kwargs = {"server_id": server_id} if enhanced_details: kwargs["params"] = {"enhanced": "true"} kwargs["scope"] = ( @@ -894,7 +922,11 @@ def flight_details( return FetchedUSSFlightDetails(v19_query=query) elif rid_version == RIDVersion.f3411_22a: query = fetch.query_and_describe( - session, "GET", url, scope=v22a.constants.Scope.DisplayProvider + session, + "GET", + url, + scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ) return FetchedUSSFlightDetails(v22a_query=query) else: @@ -946,22 +978,33 @@ def all_flights( session: UTMClientSession, dss_base_url: str = "", enhanced_details: bool = False, + server_id: Optional[str] = None, ) -> FetchedFlights: t = datetime.datetime.utcnow() - isa_list = isas(area, t, t, rid_version, session, dss_base_url) + isa_list = isas(area, t, t, rid_version, session, dss_base_url, server_id=server_id) uss_flight_queries: Dict[str, FetchedUSSFlights] = {} uss_flight_details_queries: Dict[str, FetchedUSSFlightDetails] = {} for flights_url in isa_list.flights_urls: flights_for_url = uss_flights( - flights_url, area, include_recent_positions, rid_version, session + flights_url, + area, + include_recent_positions, + rid_version, + session, + server_id=server_id, ) uss_flight_queries[flights_url] = flights_for_url if get_details and flights_for_url.success: for flight in flights_for_url.flights: details = flight_details( - flights_url, flight.id, enhanced_details, rid_version, session + flights_url, + flight.id, + enhanced_details, + rid_version, + session, + server_id=server_id, ) uss_flight_details_queries[flight.id] = details @@ -1039,13 +1082,18 @@ def subscription( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", + server_id: Optional[str] = None, ) -> FetchedSubscription: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.GetSubscription] url = f"{dss_base_url}{op.path}".replace("{id}", subscription_id) return FetchedSubscription( v19_query=fetch.query_and_describe( - session, op.verb, url, scope=v19.constants.Scope.Read + session, + op.verb, + url, + scope=v19.constants.Scope.Read, + server_id=server_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -1053,7 +1101,11 @@ def subscription( url = f"{dss_base_url}{op.path}".replace("{id}", subscription_id) return FetchedSubscription( v22a_query=fetch.query_and_describe( - session, op.verb, url, scope=v22a.constants.Scope.DisplayProvider + session, + op.verb, + url, + scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ) ) else: @@ -1137,13 +1189,18 @@ def subscriptions( rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", + server_id: Optional[str] = None, ) -> FetchedSubscriptions: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.SearchSubscriptions] url = f"{dss_base_url}{op.path}?area={rid_v1.geo_polygon_string_from_s2(area)}" return FetchedSubscriptions( v19_query=fetch.query_and_describe( - session, op.verb, url, scope=v19.constants.Scope.Read + session, + op.verb, + url, + scope=v19.constants.Scope.Read, + server_id=server_id, ) ) elif rid_version == RIDVersion.f3411_22a: @@ -1151,7 +1208,11 @@ def subscriptions( url = f"{dss_base_url}{op.path}?area={rid_v2.geo_polygon_string_from_s2(area)}" return FetchedSubscriptions( v22a_query=fetch.query_and_describe( - session, op.verb, url, scope=v22a.constants.Scope.DisplayProvider + session, + op.verb, + url, + scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ) ) else: diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index 2c8184d4b0..12aa8453d7 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -99,6 +99,7 @@ def upsert_subscription( rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, subscription_version: Optional[str] = None, + server_id: Optional[str] = None, ) -> ChangedSubscription: mutation = "create" if subscription_version is None else "update" if rid_version == RIDVersion.f3411_19: @@ -126,7 +127,12 @@ def upsert_subscription( return ChangedSubscription( mutation=mutation, v19_query=fetch.query_and_describe( - utm_client, op.verb, url, json=body, scope=v19.constants.Scope.Read + utm_client, + op.verb, + url, + json=body, + scope=v19.constants.Scope.Read, + server_id=server_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -154,6 +160,7 @@ def upsert_subscription( url, json=body, scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ), ) else: @@ -167,6 +174,7 @@ def delete_subscription( subscription_version: str, rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, + server_id: Optional[str] = None, ) -> ChangedSubscription: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.DeleteSubscription] @@ -174,7 +182,11 @@ def delete_subscription( return ChangedSubscription( mutation="delete", v19_query=fetch.query_and_describe( - utm_client, op.verb, url, scope=v19.constants.Scope.Read + utm_client, + op.verb, + url, + scope=v19.constants.Scope.Read, + server_id=server_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -183,7 +195,11 @@ def delete_subscription( return ChangedSubscription( mutation="delete", v22a_query=fetch.query_and_describe( - utm_client, op.verb, url, scope=v22a.constants.Scope.DisplayProvider + utm_client, + op.verb, + url, + scope=v22a.constants.Scope.DisplayProvider, + server_id=server_id, ), ) else: @@ -238,6 +254,7 @@ def notify( isa_id: str, utm_session: infrastructure.UTMClientSession, isa: Optional[ISA] = None, + server_id: Optional[str] = None, ) -> ISAChangeNotification: # Note that optional `extents` are not specified if self.rid_version == RIDVersion.f3411_19: @@ -254,6 +271,7 @@ def notify( url, json=body, scope=v19.constants.Scope.Write, + server_id=server_id, ) ) elif self.rid_version == RIDVersion.f3411_22a: @@ -271,6 +289,7 @@ def notify( url, json=body, scope=v22a.constants.Scope.ServiceProvider, + server_id=server_id, ) ) else: @@ -422,6 +441,7 @@ def put_isa( rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, isa_version: Optional[str] = None, + server_id: Optional[str] = None, ) -> ISAChange: mutation = "create" if isa_version is None else "update" if rid_version == RIDVersion.f3411_19: @@ -445,7 +465,12 @@ def put_isa( dss_response = ChangedISA( mutation=mutation, v19_query=fetch.query_and_describe( - utm_client, op.verb, url, json=body, scope=v19.constants.Scope.Write + utm_client, + op.verb, + url, + json=body, + scope=v19.constants.Scope.Write, + server_id=server_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -477,6 +502,7 @@ def put_isa( url, json=body, scope=v22a.constants.Scope.ServiceProvider, + server_id=server_id, ), ) else: @@ -499,6 +525,7 @@ def delete_isa( isa_version: str, rid_version: RIDVersion, utm_client: infrastructure.UTMClientSession, + server_id: Optional[str] = None, ) -> ISAChange: if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.DeleteIdentificationServiceArea] @@ -506,7 +533,11 @@ def delete_isa( dss_response = ChangedISA( mutation="delete", v19_query=fetch.query_and_describe( - utm_client, op.verb, url, scope=v19.constants.Scope.Write + utm_client, + op.verb, + url, + scope=v19.constants.Scope.Write, + server_id=server_id, ), ) elif rid_version == RIDVersion.f3411_22a: @@ -515,7 +546,11 @@ def delete_isa( dss_response = ChangedISA( mutation="delete", v22a_query=fetch.query_and_describe( - utm_client, op.verb, url, scope=v22a.constants.Scope.ServiceProvider + utm_client, + op.verb, + url, + scope=v22a.constants.Scope.ServiceProvider, + server_id=server_id, ), ) else: diff --git a/monitoring/monitorlib/mutate/scd.py b/monitoring/monitorlib/mutate/scd.py index 9f2b32da85..378dc8ca90 100644 --- a/monitoring/monitorlib/mutate/scd.py +++ b/monitoring/monitorlib/mutate/scd.py @@ -55,6 +55,7 @@ def put_subscription( min_alt_m: float = 0, max_alt_m: float = 3048, version: Optional[str] = None, + server_id: Optional[str] = None, ) -> MutatedSubscription: body = { "extents": scd.make_vol4( @@ -72,18 +73,25 @@ def put_subscription( if version: url += f"/{version}" result = MutatedSubscription( - fetch.query_and_describe(utm_client, "PUT", url, json=body, scope=scd.SCOPE_SC) + fetch.query_and_describe( + utm_client, "PUT", url, json=body, scope=scd.SCOPE_SC, server_id=server_id + ) ) result.mutation = "update" if version else "create" return result def delete_subscription( - utm_client: infrastructure.UTMClientSession, subscription_id: str, version: str + utm_client: infrastructure.UTMClientSession, + subscription_id: str, + version: str, + server_id: Optional[str] = None, ) -> MutatedSubscription: url = f"/dss/v1/subscriptions/{subscription_id}/{version}" result = MutatedSubscription( - fetch.query_and_describe(utm_client, "DELETE", url, scope=scd.SCOPE_SC) + fetch.query_and_describe( + utm_client, "DELETE", url, scope=scd.SCOPE_SC, server_id=server_id + ) ) result.mutation = "delete" return result diff --git a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml index 3e6c76b301..c5aa275e94 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml @@ -32,6 +32,7 @@ net_rid: service_providers: - participant_id: uss2 injection_base_url: http://v19.ridsp.uss2.localutm/ridsp/injection + local_debug: true netrid_service_providers_v22a: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.NetRIDServiceProviders @@ -41,6 +42,7 @@ net_rid: service_providers: - participant_id: uss1 injection_base_url: http://v22a.ridsp.uss1.localutm/ridsp/injection + local_debug: true netrid_observers_v19: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.NetRIDObserversResource @@ -50,6 +52,7 @@ net_rid: observers: - participant_id: uss3 observation_base_url: http://v19.riddp.uss3.localutm/riddp/observation + local_debug: true netrid_observers_v22a: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.netrid.NetRIDObserversResource @@ -59,6 +62,7 @@ net_rid: observers: - participant_id: uss1 observation_base_url: http://v22a.riddp.uss1.localutm/riddp/observation + local_debug: true netrid_dss_instances_v19: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.astm.f3411.DSSInstancesResource @@ -101,6 +105,7 @@ f3548: # uss1 is the mock_uss directly exposing scdsc functionality - participant_id: uss1 injection_base_url: http://scdsc.uss1.localutm/scdsc + local_debug: true # The atproxy has been disabled since it is suspected to be responsible of issue #28. Replaced by another simple mock_uss # # uss2 uses atproxy, with requests being fulfilled by mock_uss with atproxy_client functionality enabled @@ -110,6 +115,7 @@ f3548: # uss2 is the mock_uss directly exposing scdsc functionality - participant_id: uss2 injection_base_url: http://scdsc.uss2.localutm/scdsc + local_debug: true dss: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.astm.f3548.v21.DSSInstanceResource @@ -130,6 +136,7 @@ f3548_single_scenario: flight_planner: participant_id: uss1 injection_base_url: http://scdsc.uss1.localutm/scdsc + local_debug: true uss2: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.flight_planning.FlightPlannerResource @@ -139,3 +146,4 @@ f3548_single_scenario: flight_planner: participant_id: uss2 injection_base_url: http://scdsc.uss2.localutm/scdsc + local_debug: true diff --git a/monitoring/uss_qualifier/resources/astm/f3411/dss.py b/monitoring/uss_qualifier/resources/astm/f3411/dss.py index 748eb83f18..a468df5693 100644 --- a/monitoring/uss_qualifier/resources/astm/f3411/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3411/dss.py @@ -25,8 +25,20 @@ class DSSInstanceSpecification(ImplicitDict): has_private_address: Optional[bool] """Whether this DSS instance is expected to have a private address that is not publicly addressable.""" + local_debug: Optional[bool] + """Whether this DSS instance is running locally for debugging or development purposes. Mostly used for relaxing + constraints around encryption. + Assumed to be true if left unspecified and has_private_address is true, otherwise defaults to false + """ + def __init__(self, *args, **kwargs): super().__init__(**kwargs) + if ( + not self.has_field_with_value("local_debug") + and self.has_field_with_value("has_private_address") + and self.get("has_private_address") + ): + self.local_debug = True try: urlparse(self.base_url) except ValueError: @@ -38,6 +50,7 @@ class DSSInstance(object): rid_version: RIDVersion base_url: str has_private_address: bool = False + local_debug: bool = False client: infrastructure.UTMClientSession def __init__( @@ -45,6 +58,7 @@ def __init__( participant_id: ParticipantID, base_url: str, has_private_address: Optional[bool], + local_debug: Optional[bool], rid_version: RIDVersion, auth_adapter: infrastructure.AuthAdapter, ): @@ -55,6 +69,8 @@ def __init__( if has_private_address is not None: self.has_private_address = has_private_address + if local_debug is not None: + self.local_debug = local_debug class DSSInstanceResource(Resource[DSSInstanceSpecification]): @@ -67,6 +83,7 @@ def __init__( specification.participant_id, specification.base_url, specification.has_private_address, + specification.local_debug, specification.rid_version, auth_adapter.adapter, ) @@ -95,6 +112,7 @@ def __init__( s.participant_id, s.base_url, s.has_private_address, + s.local_debug, s.rid_version, auth_adapter.adapter, ) diff --git a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py index a5df2e0df2..a9e8b923ad 100644 --- a/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py @@ -53,7 +53,12 @@ def find_op_intent( url = "/dss/v1/operational_intent_references/query" req = QueryOperationalIntentReferenceParameters(area_of_interest=extent) query = fetch.query_and_describe( - self.client, "POST", url, scope=SCOPE_SC, json=req + self.client, + "POST", + url, + scope=SCOPE_SC, + json=req, + server_id=self.participant_id, ) if query.status_code != 200: result = None @@ -67,7 +72,9 @@ def get_full_op_intent( self, op_intent_ref: OperationalIntentReference ) -> Tuple[OperationalIntent, fetch.Query]: url = f"{op_intent_ref.uss_base_url}/uss/v1/operational_intents/{op_intent_ref.id}" - query = fetch.query_and_describe(self.client, "GET", url, scope=SCOPE_SC) + query = fetch.query_and_describe( + self.client, "GET", url, scope=SCOPE_SC, server_id=self.participant_id + ) if query.status_code != 200: result = None else: diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py index 4749cf0e55..96aee599ab 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py @@ -90,7 +90,12 @@ def request_flight( url = "{}/v1/flights/{}".format(self.config.injection_base_url, flight_id) query = fetch.query_and_describe( - self.client, "PUT", url, json=request, scope=SCOPE_SCD_QUALIFIER_INJECT + self.client, + "PUT", + url, + json=request, + scope=SCOPE_SCD_QUALIFIER_INJECT, + server_id=self.config.participant_id, ) if query.status_code != 200: raise QueryError( @@ -116,7 +121,11 @@ def cleanup_flight( ) -> Tuple[DeleteFlightResponse, fetch.Query]: url = "{}/v1/flights/{}".format(self.config.injection_base_url, flight_id) query = fetch.query_and_describe( - self.client, "DELETE", url, scope=SCOPE_SCD_QUALIFIER_INJECT + self.client, + "DELETE", + url, + scope=SCOPE_SCD_QUALIFIER_INJECT, + server_id=self.config.participant_id, ) if query.status_code != 200: raise QueryError( @@ -139,7 +148,11 @@ def cleanup_flight( def get_target_information(self) -> FlightPlannerInformation: url_status = "{}/v1/status".format(self.config.injection_base_url) version_query = fetch.query_and_describe( - self.client, "GET", url_status, scope=SCOPE_SCD_QUALIFIER_INJECT + self.client, + "GET", + url_status, + scope=SCOPE_SCD_QUALIFIER_INJECT, + server_id=self.config.participant_id, ) if version_query.status_code != 200: raise QueryError( @@ -159,7 +172,11 @@ def get_target_information(self) -> FlightPlannerInformation: url_capabilities = "{}/v1/capabilities".format(self.config.injection_base_url) capabilities_query = fetch.query_and_describe( - self.client, "GET", url_capabilities, scope=SCOPE_SCD_QUALIFIER_INJECT + self.client, + "GET", + url_capabilities, + scope=SCOPE_SCD_QUALIFIER_INJECT, + server_id=self.config.participant_id, ) if capabilities_query.status_code != 200: raise QueryError( @@ -187,7 +204,12 @@ def clear_area(self, extent: Volume4D) -> Tuple[ClearAreaResponse, fetch.Query]: req = ClearAreaRequest(request_id=str(uuid.uuid4()), extent=extent) url = f"{self.config.injection_base_url}/v1/clear_area_requests" query = fetch.query_and_describe( - self.client, "POST", url, scope=SCOPE_SCD_QUALIFIER_INJECT, json=req + self.client, + "POST", + url, + scope=SCOPE_SCD_QUALIFIER_INJECT, + json=req, + server_id=self.config.participant_id, ) if query.status_code != 200: raise QueryError( diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss.py b/monitoring/uss_qualifier/resources/interuss/mock_uss.py index 8922a6e1e5..c0a2df3b1b 100644 --- a/monitoring/uss_qualifier/resources/interuss/mock_uss.py +++ b/monitoring/uss_qualifier/resources/interuss/mock_uss.py @@ -25,7 +25,11 @@ def __init__( def get_status(self) -> fetch.Query: return fetch.query_and_describe( - self.session, "GET", "/scdsc/v1/status", scope=SCOPE_SCD_QUALIFIER_INJECT + self.session, + "GET", + "/scdsc/v1/status", + scope=SCOPE_SCD_QUALIFIER_INJECT, + server_id=self.participant_id, ) # TODO: Add other methods to interact with the mock USS in other ways (like starting/stopping message signing data collection) diff --git a/monitoring/uss_qualifier/resources/netrid/observers.py b/monitoring/uss_qualifier/resources/netrid/observers.py index da330d3651..da9bceda82 100644 --- a/monitoring/uss_qualifier/resources/netrid/observers.py +++ b/monitoring/uss_qualifier/resources/netrid/observers.py @@ -16,16 +16,19 @@ class RIDSystemObserver(object): participant_id: str base_url: str session: infrastructure.UTMClientSession + local_debug: bool def __init__( self, participant_id: str, base_url: str, auth_adapter: infrastructure.AuthAdapter, + local_debug: bool, ): self.session = UTMClientSession(base_url, auth_adapter) self.participant_id = participant_id self.base_url = base_url + self.local_debug = local_debug # TODO: Change observation API to use an InterUSS scope rather than re-using an ASTM scope self.rid_version = RIDVersion.f3411_19 @@ -40,7 +43,11 @@ def observe_system( rect.hi().lng().degrees, ) query = fetch.query_and_describe( - self.session, "GET", url, scope=self.rid_version.read_scope + self.session, + "GET", + url, + scope=self.rid_version.read_scope, + server_id=self.participant_id, ) try: result = ( @@ -59,7 +66,10 @@ def observe_flight_details( self, flight_id: str ) -> Tuple[Optional[observation_api.GetDetailsResponse], fetch.Query]: query = fetch.query_and_describe( - self.session, "GET", f"/display_data/{flight_id}" + self.session, + "GET", + f"/display_data/{flight_id}", + server_id=self.participant_id, ) try: result = ( @@ -81,6 +91,12 @@ class ObserverConfiguration(ImplicitDict): observation_base_url: str """Base URL for the observer's implementation of the interfaces/automated-testing/rid/observation.yaml API""" + local_debug: Optional[bool] + """Whether this Observer instance is running locally for debugging or development purposes. Mostly used for relaxing + constraints around encryption. + Assumed to be true if left unspecified and has_private_address is true, otherwise defaults to false + """ + class NetRIDObserversSpecification(ImplicitDict): observers: List[ObserverConfiguration] @@ -99,6 +115,7 @@ def __init__( o.participant_id, o.observation_base_url, auth_adapter.adapter, + o.local_debug, ) for o in specification.observers ] diff --git a/monitoring/uss_qualifier/resources/netrid/service_providers.py b/monitoring/uss_qualifier/resources/netrid/service_providers.py index f4ec02a91d..76c02a981f 100644 --- a/monitoring/uss_qualifier/resources/netrid/service_providers.py +++ b/monitoring/uss_qualifier/resources/netrid/service_providers.py @@ -1,5 +1,5 @@ import datetime -from typing import List +from typing import List, Optional from urllib.parse import urlparse from implicitdict import ImplicitDict @@ -20,6 +20,11 @@ class ServiceProviderConfiguration(ImplicitDict): injection_base_url: str """Base URL for the Service Provider's implementation of the interfaces/automated-testing/rid/injection.yaml API""" + local_debug: Optional[bool] + """Whether this Service Provider instance is running locally for debugging or development purposes. Mostly used for relaxing + constraints around encryption. + """ + def __init__(self, *args, **kwargs): super().__init__(**kwargs) try: @@ -38,16 +43,19 @@ class NetRIDServiceProvider(object): participant_id: str base_url: str client: infrastructure.UTMClientSession + local_debug: bool def __init__( self, participant_id: str, base_url: str, auth_adapter: infrastructure.AuthAdapter, + local_debug: bool, ): self.participant_id = participant_id self.base_url = base_url self.client = infrastructure.UTMClientSession(base_url, auth_adapter) + self.local_debug = local_debug def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Query: return fetch.query_and_describe( @@ -56,6 +64,7 @@ def submit_test(self, request: CreateTestParameters, test_id: str) -> fetch.Quer url=f"/tests/{test_id}", json=request, scope=SCOPE_RID_QUALIFIER_INJECT, + server_id=self.participant_id, ) def delete_test(self, test_id: str, version: str) -> fetch.Query: @@ -64,6 +73,7 @@ def delete_test(self, test_id: str, version: str) -> fetch.Query: "DELETE", url=f"/tests/{test_id}/{version}", scope=SCOPE_RID_QUALIFIER_INJECT, + server_id=self.participant_id, ) @@ -77,7 +87,10 @@ def __init__( ): self.service_providers = [ NetRIDServiceProvider( - s.participant_id, s.injection_base_url, auth_adapter.adapter + s.participant_id, + s.injection_base_url, + auth_adapter.adapter, + s.get("local_debug", False), ) for s in specification.service_providers ] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index a9bbc945d3..1f4eae6a2d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -1,11 +1,13 @@ import re -from typing import List, Dict +from typing import List, Dict, Set from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import evaluation, QueryType from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance from monitoring.uss_qualifier.scenarios.interuss.evaluation_scenario import ( ReportEvaluationScenario, ) @@ -25,9 +27,11 @@ class AggregateChecks(ReportEvaluationScenario): _rid_version: RIDVersion _service_providers: List[NetRIDServiceProvider] _observers: List[RIDSystemObserver] + _dss_instances: List[DSSInstance] _queries: List[fetch.Query] _participants_by_base_url: Dict[str, ParticipantID] = dict() + _debug_mode_usses: Set[ParticipantID] = set() _queries_by_participant: Dict[ParticipantID, List[fetch.Query]] def __init__( @@ -35,11 +39,13 @@ def __init__( report_resource: TestSuiteReportResource, service_providers: NetRIDServiceProviders, observers: NetRIDObserversResource, + dss_instances: DSSInstancesResource, ): super().__init__(report_resource) self._queries = self.report.queries() self._service_providers = service_providers.service_providers self._observers = observers.observers + self._dss_instances = dss_instances.dss_instances # identify SPs and observers by their base URL self._participants_by_base_url.update( @@ -49,6 +55,19 @@ def __init__( {dp.base_url: dp.participant_id for dp in self._observers} ) + # identify usses running in debug mode + for sp in self._service_providers: + if sp.local_debug: + self._debug_mode_usses.add(sp.participant_id) + + for o in self._observers: + if o.local_debug: + self._debug_mode_usses.add(o.participant_id) + + for dss in self._dss_instances: + if dss.local_debug: + self._debug_mode_usses.add(dss.participant_id) + # collect and classify queries by participant self._queries_by_participant = { participant: list() @@ -58,6 +77,8 @@ def __init__( for base_url, participant in self._participants_by_base_url.items(): if query.request.url.startswith(base_url): self._queries_by_participant[participant].append(query) + # TODO opportunity here to set the participant_id on the query if it's not already there + # maybe do so after most/all queries have been tagged at the call site where possible break # Only consider queries with the participant/server explicitly identified @@ -85,6 +106,8 @@ def run(self): "observer", f"configured observer: {o.participant_id} - {o.base_url}" ) + self.record_note("debug_usses", f"debug mode usses: {self._debug_mode_usses}") + # DP performance self.begin_test_case("Performance of Display Providers requests") self.begin_test_step("Performance of /display_data requests") @@ -103,8 +126,71 @@ def run(self): self.end_test_step() self.end_test_case() + self.begin_test_case("Verify https is in use") + self.begin_test_step("Verify https is in use") + + self._verify_https_everywhere() + + self.end_test_step() + self.end_test_case() + self.end_test_scenario() + def _verify_https_everywhere(self): + + for participant_id, participant_queries in self._queries_by_participant.items(): + self._inspect_participant_queries(participant_id, participant_queries) + + # Check that all queries have been attributed to a participant + unattr_queries = [ + query.request.url + for query in self._queries + if query.get("server_id") is None + ] + if len(unattr_queries) > 0: + # TODO clean this up: this is an internal requirement and not a check, + # leaving as-is during development to make sure the test-suite runs but we know about unattributed queries + # ultimately this check could go into the constructor and blow things up early + self.record_note( + "unattributed_queries", + f"found {len(unattr_queries)} unattributed queries", + ) + with self.check( + "All interactions happen over https", + [], + ) as check: + check.record_failed( + f"found unattributed queries: {unattr_queries}", Severity.Medium + ) + + def _inspect_participant_queries( + self, participant_id: str, participant_queries: List[fetch.Query] + ): + found_cleartext_query = False + for query in participant_queries: + if query.request.url.startswith("http://"): + found_cleartext_query = True + if participant_id not in self._debug_mode_usses: + self.record_note( + "cleartext-query", + f"query is not https: {participant_id} - {query.request.url}", + ) + + if participant_id not in self._debug_mode_usses: + with self.check( + "All interactions happen over https", + [participant_id], + ) as check: + if found_cleartext_query: + check.record_failed( + "found at least one cleartext http query", Severity.Medium + ) + else: + self.record_note( + "https-check", + f"participant {participant_id} is in local debug mode, skipping HTTPS check", + ) + def _sp_flights_area_times_step(self): for participant, all_queries in self._queries_by_participant.items(): # identify successful flights queries diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index 0a583a954d..5a9fbf28cb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -51,7 +51,10 @@ def _setup_case(self): def _delete_isa_if_exists(self): fetched = fetch.isa( - self._isa_id, rid_version=self._dss.rid_version, session=self._dss.client + self._isa_id, + rid_version=self._dss.rid_version, + session=self._dss.client, + server_id=self._dss.participant_id, ) self.record_query(fetched.query) with self.check("Successful ISA query", [self._dss.participant_id]) as check: @@ -69,6 +72,7 @@ def _delete_isa_if_exists(self): fetched.isa.version, self._dss.rid_version, self._dss.client, + server_id=self._dss.participant_id, ) self.record_query(deleted.dss_query.query) for subscriber_id, notification in deleted.notifications.items(): @@ -127,6 +131,7 @@ def _create_isa_step(self): isa_version=None, alt_lo=self._isa.altitude_min, alt_hi=self._isa.altitude_max, + server_id=self._dss.participant_id, ) self.record_query(isa_change.dss_query.query) for notification_query in isa_change.notifications.values(): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index c069a0beaa..0e4ae19dae 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -231,6 +231,7 @@ def evaluate_system_instantaneously( get_details=True, rid_version=self._rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) for q in sp_observation.queries: self._test_scenario.record_query(q) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 8b9cfea7dd..3c85f041e1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -107,6 +107,7 @@ def get_isa( isa_id=isa_id, rid_version=self._dss.rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result(check, isa, f"Failed to get ISA {isa_id}") @@ -160,6 +161,7 @@ def put_isa( isa_version=isa_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -223,6 +225,7 @@ def del_isa( isa_version=isa_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -261,6 +264,7 @@ def cleanup_isa( isa_id=isa_id, rid_version=self._dss.rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -275,6 +279,7 @@ def cleanup_isa( isa_version=isa.isa.version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -309,6 +314,7 @@ def search_subs( area=area, rid_version=self._dss.rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -339,6 +345,7 @@ def get_sub( subscription_id=sub_id, rid_version=self._dss.rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -378,6 +385,7 @@ def no_sub( subscription_id=sub_id, rid_version=self._dss.rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -420,6 +428,7 @@ def put_sub_expect_response_code( subscription_version=sub_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -468,6 +477,7 @@ def put_sub( subscription_version=sub_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -499,6 +509,7 @@ def del_sub( subscription_version=sub_version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -537,6 +548,7 @@ def cleanup_sub( subscription_id=sub_id, rid_version=self._dss.rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( @@ -555,6 +567,7 @@ def cleanup_sub( subscription_version=sub.subscription.version, rid_version=self._dss.rid_version, utm_client=self._dss.client, + server_id=self._dss.participant_id, ) self._handle_query_result( diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 8dcbdfead6..51a4c84e15 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -15,6 +15,9 @@ The service providers to evaluate in the report. ### observers The observers to evaluate in the report. +### dss_instances +The DSS instances that have been relied upon or tested by the framework. + ## Performance of Display Providers requests test case ### Performance of /display_data requests test step @@ -42,3 +45,11 @@ of the durations for the subsequent display data queries do not exceed the respe **[astm.f3411.v19.NET0260-a](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles of the durations for the replies to requested flights in an area do not exceed the respective thresholds `NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). + +## Verify https is in use test case + +### Verify https is in use test step + +#### All interactions happen over https check + +If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.py index 0dc608f7f0..7744c3056a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.py @@ -1,4 +1,5 @@ from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource from monitoring.uss_qualifier.resources.netrid import ( NetRIDServiceProviders, @@ -16,6 +17,7 @@ def __init__( report_resource: TestSuiteReportResource, service_providers: NetRIDServiceProviders, observers: NetRIDObserversResource, + dss_instances: DSSInstancesResource, ): - super().__init__(report_resource, service_providers, observers) + super().__init__(report_resource, service_providers, observers, dss_instances) self._rid_version = RIDVersion.f3411_19 diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 17760ea7eb..a6c5d7e6fa 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -16,6 +16,9 @@ The service providers to evaluate in the report. ### observers The observers to evaluate in the report. +### dss_instances +The DSS instances that have been relied upon or tested by the framework. + ## Performance of Display Providers requests test case ### Performance of /display_data requests test step @@ -43,3 +46,11 @@ of the durations for the subsequent display data queries do not exceed the respe **[astm.f3411.v22a.NET0260-a](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles of the durations for the replies to requested flights in an area do not exceed the respective thresholds `NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). + +## Verify https is in use test case + +### Verify https is in use test step + +#### All interactions happen over https check + +If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.py index 53dc241cb4..5da19d53e4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.py @@ -1,4 +1,5 @@ from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource from monitoring.uss_qualifier.resources.netrid import ( NetRIDServiceProviders, @@ -16,6 +17,7 @@ def __init__( report_resource: TestSuiteReportResource, service_providers: NetRIDServiceProviders, observers: NetRIDObserversResource, + dss_instances: DSSInstancesResource, ): - super().__init__(report_resource, service_providers, observers) + super().__init__(report_resource, service_providers, observers, dss_instances) self._rid_version = RIDVersion.f3411_22a diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml index 930fd79b59..51b49b5150 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml @@ -55,6 +55,7 @@ report_evaluation_scenario: resources: service_providers: service_providers observers: observers + dss_instances: dss_instances participant_verifiable_capabilities: - id: service_provider name: NetRID Service Provider diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml index f113125c71..506cfc6b02 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml @@ -63,6 +63,7 @@ report_evaluation_scenario: resources: service_providers: service_providers observers: observers + dss_instances: dss_instances participant_verifiable_capabilities: - id: service_provider name: NetRID Service Provider