From 6f9141a1ba9591d3a8ec32ab8949afdb530da0df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Thu, 14 Sep 2023 11:26:04 +0200 Subject: [PATCH 1/9] [uss_qualifier/netrid/dss/isa_simple] Refactor and complete create case --- .../mock_uss/riddp/routes_observation.py | 4 +- monitoring/mock_uss/tracer/tracer_poll.py | 4 +- monitoring/monitorlib/fetch/rid.py | 10 +- monitoring/monitorlib/geo.py | 13 ++ monitoring/monitorlib/mutate/rid.py | 33 +++- monitoring/monitorlib/rid.py | 13 +- monitoring/monitorlib/schema_validation.py | 6 + .../configurations/dev/library/resources.yaml | 15 +- .../resources/netrid/service_area.py | 26 ++- .../astm/netrid/common/dss/isa_simple.py | 183 ++++++------------ .../common/dss/subscription_validation.py | 4 +- .../netrid/common/dss_interoperability.py | 4 +- .../scenarios/astm/netrid/dss_wrapper.py | 83 ++++++-- .../astm/netrid/v22a/dss/isa_simple.md | 29 +-- 14 files changed, 249 insertions(+), 178 deletions(-) diff --git a/monitoring/mock_uss/riddp/routes_observation.py b/monitoring/mock_uss/riddp/routes_observation.py index d4a1f3b5e6..3a1c1c8c4b 100644 --- a/monitoring/mock_uss/riddp/routes_observation.py +++ b/monitoring/mock_uss/riddp/routes_observation.py @@ -92,7 +92,9 @@ def riddp_display_data() -> Tuple[str, int]: # Get ISAs in the DSS t = arrow.utcnow().datetime - isa_list: FetchedISAs = fetch.isas(view, t, t, rid_version, utm_client) + isa_list: FetchedISAs = fetch.isas( + geo.get_latlngrect_vertices(view), t, t, rid_version, utm_client + ) if not isa_list.success: msg = f"Error fetching ISAs from DSS: {isa_list.errors}" logger.error(msg) diff --git a/monitoring/mock_uss/tracer/tracer_poll.py b/monitoring/mock_uss/tracer/tracer_poll.py index 0d9eb7e5eb..4e29c6a8d0 100755 --- a/monitoring/mock_uss/tracer/tracer_poll.py +++ b/monitoring/mock_uss/tracer/tracer_poll.py @@ -21,7 +21,7 @@ from monitoring.mock_uss.tracer import context from monitoring.monitorlib.fetch.rid import FetchedISAs from monitoring.monitorlib.fetch.scd import FetchedEntities -from monitoring.monitorlib.geo import make_latlng_rect +from monitoring.monitorlib.geo import make_latlng_rect, get_latlngrect_vertices from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.multiprocessing import SynchronizedValue @@ -97,7 +97,7 @@ def poll_observation_areas() -> None: def poll_isas(area: ObservationArea, logger: tracerlog.Logger) -> None: rid_client = context.get_client(area.f3411.auth_spec, area.f3411.dss_base_url) - box = make_latlng_rect(area.area.volume) + box = get_latlngrect_vertices(make_latlng_rect(area.area.volume)) log_name = "poll_isas" t0 = datetime.datetime.utcnow() diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index 3da7beb610..5415a306cc 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -654,7 +654,7 @@ def has_different_content_than(self, other: Any) -> bool: def isas( - box: s2sphere.LatLngRect, + area: List[s2sphere.LatLng], start_time: datetime.datetime, end_time: datetime.datetime, rid_version: RIDVersion, @@ -665,7 +665,7 @@ def isas( t1 = rid_version.format_time(end_time) if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.SearchIdentificationServiceAreas] - area = rid_v1.geo_polygon_string_from_s2(geo.get_latlngrect_vertices(box)) + area = rid_v1.geo_polygon_string_from_s2(area) url = f"{dss_base_url}{op.path}?area={area}&earliest_time={t0}&latest_time={t1}" return FetchedISAs( v19_query=fetch.query_and_describe( @@ -674,7 +674,7 @@ def isas( ) elif rid_version == RIDVersion.f3411_22a: op = v22a.api.OPERATIONS[v22a.api.OperationID.SearchIdentificationServiceAreas] - area = rid_v2.geo_polygon_string_from_s2(geo.get_latlngrect_vertices(box)) + area = rid_v2.geo_polygon_string_from_s2(area) url = f"{dss_base_url}{op.path}?area={area}&earliest_time={t0}&latest_time={t1}" return FetchedISAs( v22a_query=fetch.query_and_describe( @@ -948,7 +948,9 @@ def all_flights( enhanced_details: bool = False, ) -> FetchedFlights: t = datetime.datetime.utcnow() - isa_list = isas(area, t, t, rid_version, session, dss_base_url) + isa_list = isas( + geo.get_latlngrect_vertices(area), t, t, rid_version, session, dss_base_url + ) uss_flight_queries: Dict[str, FetchedUSSFlights] = {} uss_flight_details_queries: Dict[str, FetchedUSSFlightDetails] = {} diff --git a/monitoring/monitorlib/geo.py b/monitoring/monitorlib/geo.py index dc191d5a6a..f921552bc0 100644 --- a/monitoring/monitorlib/geo.py +++ b/monitoring/monitorlib/geo.py @@ -251,3 +251,16 @@ def to_vertices(self) -> List[s2sphere.LatLng]: s2sphere.LatLng.from_degrees(self.lat_max, self.lng_max), s2sphere.LatLng.from_degrees(self.lat_min, self.lng_max), ] + + +class LatLngVertex(ImplicitDict): + """Vertex in latitude and longitude""" + + lat: float + """Latitude (degrees)""" + + lng: float + """Longitude (degrees)""" + + def as_s2sphere(self) -> s2sphere.LatLng: + return s2sphere.LatLng.from_degrees(self.lat, self.lng) diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index 2c8184d4b0..6f6c98ca4f 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -14,7 +14,13 @@ import yaml from yaml.representer import Representer -from monitoring.monitorlib import fetch, infrastructure, rid_v1, rid_v2 +from monitoring.monitorlib import ( + fetch, + infrastructure, + rid_v1, + rid_v2, + schema_validation, +) class ChangedSubscription(RIDQuery): @@ -61,6 +67,8 @@ def errors(self) -> List[str]: f"Error parsing F3411-22a USS PutSubscriptionResponse: {str(e)}" ] + # TODO: add schema validation (like ChangedISA) + return [] @property @@ -308,7 +316,11 @@ def _v22a_response( @property def errors(self) -> List[str]: - if self.status_code != 200: + # Tolerate reasonable-but-technically-incorrect code 201 + if not ( + self.status_code == 200 + or (self.mutation == "create" and self.status_code == 201) + ): return ["Failed to mutate ISA ({})".format(self.status_code)] if self.query.response.json is None: return ["ISA response did not include valid JSON"] @@ -337,6 +349,23 @@ def errors(self) -> List[str]: f"Error parsing F3411-22a USS PutIdentificationServiceAreaResponse: {str(e)}" ] + # TODO: This validates the schema only for v22a as the v19 OpenAPI definition contains a mistake in + # 'components.schemas.SubscriptionState', the non-existing property 'subscription' is marked as required. + # The augmented.yaml version of the definition should be fixed and then this can be applied to both version. + if self.rid_version == RIDVersion.f3411_22a: + validation_errors = schema_validation.validate( + self.rid_version.openapi_path, + self.rid_version.openapi_delete_isa_response_path + if self.mutation == "delete" + else self.rid_version.openapi_put_isa_response_path, + self.query.response.json, + ) + if validation_errors: + return [ + f"PUT ISA response JSON validation error: [{e.json_path}] {e.message}" + for e in validation_errors + ] + return [] @property diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index 196409a752..22a612b3b8 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -5,6 +5,7 @@ from monitoring.monitorlib import schema_validation from uas_standards.astm.f3411 import v19, v22a +import uas_standards.astm.f3411.v19.api import uas_standards.astm.f3411.v19.constants import uas_standards.astm.f3411.v22a.api import uas_standards.astm.f3411.v22a.constants @@ -56,6 +57,15 @@ def openapi_put_isa_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") + @property + def openapi_delete_isa_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.DeleteIdentificationServiceAreaResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.DeleteIdentificationServiceAreaResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + @property def realtime_period(self) -> timedelta: if self == RIDVersion.f3411_19: @@ -202,7 +212,8 @@ def dss_max_subscriptions_per_area(self) -> int: def flights_url_of(self, base_url: str) -> str: if self == RIDVersion.f3411_19: - return base_url + flights_path = v19.api.OPERATIONS[v19.api.OperationID.SearchFlights].path + return base_url + flights_path elif self == RIDVersion.f3411_22a: flights_path = v22a.api.OPERATIONS[v22a.api.OperationID.SearchFlights].path return base_url + flights_path diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index 29346ed7cb..56ff259f62 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -19,6 +19,9 @@ class F3411_19(str, Enum): PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) + DeleteIdentificationServiceAreaResponse = ( + "components.schemas.DeleteIdentificationServiceAreaResponse" + ) class F3411_22a(str, Enum): @@ -28,6 +31,9 @@ class F3411_22a(str, Enum): PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) + DeleteIdentificationServiceAreaResponse = ( + "components.schemas.DeleteIdentificationServiceAreaResponse" + ) class F3548_21(str, Enum): diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index e299b45688..324a0b5d61 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -32,12 +32,19 @@ net_rid: specification: base_url: https://uss_qualifier.test.utm/dummy_base_url footprint: - lat_min: 37.1853 - lng_min: -80.6140 - lat_max: 37.2148 - lng_max: -80.5440 + - lat: 37.1853 + lng: -80.6140 + - lat: 37.2148 + lng: -80.6140 + - lat: 37.2148 + lng: -80.5440 + - lat: 37.1853 + lng: -80.5440 altitude_min: 0 altitude_max: 3048 + reference_time: '2023-01-10T00:00:00.123456+00:00' + time_start: '2023-01-10T00:00:01.123456+00:00' + time_end: '2023-01-10T01:00:01.123456+00:00' net_rid_sims: adjacent_circular_flights_data: diff --git a/monitoring/uss_qualifier/resources/netrid/service_area.py b/monitoring/uss_qualifier/resources/netrid/service_area.py index d14ce141dc..9e2e01224a 100644 --- a/monitoring/uss_qualifier/resources/netrid/service_area.py +++ b/monitoring/uss_qualifier/resources/netrid/service_area.py @@ -1,5 +1,8 @@ -from implicitdict import ImplicitDict -from monitoring.monitorlib.geo import LatLngBoundingBox +import datetime +from typing import List + +from implicitdict import ImplicitDict, StringBasedDateTime +from monitoring.monitorlib.geo import LatLngVertex from monitoring.uss_qualifier.resources.resource import Resource @@ -12,7 +15,7 @@ class ServiceAreaSpecification(ImplicitDict): This URL will probably not identify a real resource in tests.""" - footprint: LatLngBoundingBox + footprint: List[LatLngVertex] """2D outline of service area""" altitude_min: float = 0 @@ -21,6 +24,23 @@ class ServiceAreaSpecification(ImplicitDict): altitude_max: float = 3048 """Upper altitude bound of service area, meters above WGS84 ellipsoid""" + reference_time: StringBasedDateTime + """Reference time used to adjust start and end times at runtime""" + + time_start: StringBasedDateTime + """Start time of service area (relative to reference_time)""" + + time_end: StringBasedDateTime + """End time of service area (relative to reference_time)""" + + def shifted_time_start(self, now: datetime.datetime) -> datetime.datetime: + dt = now - self.reference_time.datetime + return self.time_start.datetime + dt + + def shifted_time_end(self, now: datetime.datetime) -> datetime.datetime: + dt = now - self.reference_time.datetime + return self.time_end.datetime + dt + class ServiceAreaResource(Resource[ServiceAreaSpecification]): specification: ServiceAreaSpecification diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index 0a583a954d..5e1a1e4188 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -1,8 +1,7 @@ -from datetime import timedelta +from typing import Optional import arrow -from monitoring.monitorlib import schema_validation from monitoring.monitorlib.fetch import rid as fetch from monitoring.monitorlib.mutate import rid as mutate from monitoring.prober.infrastructure import register_resource_type @@ -10,12 +9,10 @@ from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource +from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario -MAX_SKEW = 1e-6 # seconds maximum difference between expected and actual timestamps - - class ISASimple(GenericTestScenario): """Based on prober/rid/v2/test_isa_simple.py from the legacy prober tool.""" @@ -28,10 +25,19 @@ def __init__( isa: ServiceAreaResource, ): super().__init__() - self._dss = dss.dss_instance + self._dss = ( + dss.dss_instance + ) # TODO: delete once _delete_isa_if_exists updated to use dss_wrapper + self._dss_wrapper = DSSWrapper(self, dss.dss_instance) self._isa_id = id_generator.id_factory.make_id(ISASimple.ISA_TYPE) + self._isa_version: Optional[str] = None self._isa = isa.specification + now = arrow.utcnow().datetime + self._isa_start_time = self._isa.shifted_time_start(now) + self._isa_end_time = self._isa.shifted_time_end(now) + self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] + def run(self): self.begin_test_scenario() @@ -45,7 +51,14 @@ def run(self): def _setup_case(self): self.begin_test_case("Setup") - self._ensure_clean_workspace_step() + def _ensure_clean_workspace_step(): + self.begin_test_step("Ensure clean workspace") + + self._delete_isa_if_exists() + + self.end_test_step() + + _ensure_clean_workspace_step() self.end_test_case() @@ -94,135 +107,57 @@ def _delete_isa_if_exists(self): query_timestamps=[notification.query.request.timestamp], ) - def _ensure_clean_workspace_step(self): - self.begin_test_step("Ensure clean workspace") + def _get_isa_by_id_step(self): + self.begin_test_step("Get ISA by ID") - self._delete_isa_if_exists() + with self.check( + "Successful ISA query", [self._dss_wrapper.participant_id] + ) as check: + fetched = self._dss_wrapper.get_isa(check, self._isa_id) + + with self.check( + "ISA version match", [self._dss_wrapper.participant_id] + ) as check: + if ( + self._isa_version is not None + and fetched.isa.version != self._isa_version + ): + check.record_failed( + "DSS returned ISA with incorrect version", + Severity.High, + f"DSS should have returned an ISA with the version {self._isa_version}, but instead the ISA returned had the version {fetched.isa.version}", + query_timestamps=[fetched.query.request.timestamp], + ) self.end_test_step() def _create_and_check_isa_case(self): self.begin_test_case("Create and check ISA") - self._create_isa_step() - - # TODO: Get ISA by ID - - self.end_test_case() - - def _create_isa_step(self): - self.begin_test_step("Create ISA") - - start_time = arrow.utcnow().datetime + timedelta(seconds=1) - end_time = start_time + timedelta(minutes=60) - area = self._isa.footprint.to_vertices() - isa_change = mutate.put_isa( - area_vertices=area, - start_time=start_time, - end_time=end_time, - uss_base_url=self._isa.base_url, - isa_id=self._isa_id, - rid_version=self._dss.rid_version, - utm_client=self._dss.client, - isa_version=None, - alt_lo=self._isa.altitude_min, - alt_hi=self._isa.altitude_max, - ) - self.record_query(isa_change.dss_query.query) - for notification_query in isa_change.notifications.values(): - self.record_query(notification_query.query) - t_dss = isa_change.dss_query.query.request.timestamp - - with self.check("ISA created", [self._dss.participant_id]) as check: - if isa_change.dss_query.status_code == 200: - check.record_passed() - elif isa_change.dss_query.status_code == 201: - check.record_failed( - f"PUT ISA returned technically-incorrect 201", - Severity.Low, - "DSS should return 200 from PUT ISA, but instead returned the reasonable-but-technically-incorrect code 201", - query_timestamps=[t_dss], - ) - else: - check.record_failed( - f"PUT ISA returned {isa_change.dss_query.status_code}", - Severity.High, - f"DSS should return 200 from PUT ISA, but instead returned {isa_change.dss_query.status_code}", - query_timestamps=[t_dss], + def _create_isa_step(): + self.begin_test_step("Create ISA") + + with self.check("ISA created", [self._dss.participant_id]) as check: + isa_change = self._dss_wrapper.put_isa( + check, + area_vertices=self._isa_area, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, ) + self._isa_version = isa_change.dss_query.isa.version - with self.check("ISA ID matches", [self._dss.participant_id]) as check: - if isa_change.dss_query.isa.id != self._isa_id: - check.record_failed( - f"PUT ISA returned ISA with incorrect ID", - Severity.High, - f"DSS should have recorded and returned the ISA ID {self._isa_id} as requested in the path, but response body instead specified {isa_change.dss_query.isa.id}", - query_timestamps=[t_dss], - ) - with self.check("ISA URL matches", [self._dss.participant_id]) as check: - expected_flights_url = self._dss.rid_version.flights_url_of( - self._isa.base_url - ) - actual_flights_url = isa_change.dss_query.isa.flights_url - if actual_flights_url != expected_flights_url: - check.record_failed( - f"PUT ISA returned ISA with incorrect URL", - Severity.High, - f"DSS should have returned an ISA with a flights URL of {expected_flights_url}, but instead the ISA returned had a flights URL of {actual_flights_url}", - query_timestamps=[t_dss], - ) - with self.check("ISA start time matches", [self._dss.participant_id]) as check: - if ( - abs((isa_change.dss_query.isa.time_start - start_time).total_seconds()) - > MAX_SKEW - ): - check.record_failed( - "PUT ISA returned ISA with incorrect start time", - Severity.High, - f"DSS should have returned an ISA with a start time of {start_time}, but instead the ISA returned had a start time of {isa_change.dss_query.isa.time_start}", - query_timestamps=[t_dss], - ) - with self.check("ISA end time matches", [self._dss.participant_id]) as check: - if ( - abs((isa_change.dss_query.isa.time_end - end_time).total_seconds()) - > MAX_SKEW - ): - check.record_failed( - "PUT ISA returned ISA with incorrect end time", - Severity.High, - f"DSS should have returned an ISA with an end time of {end_time}, but instead the ISA returned had an end time of {isa_change.dss_query.isa.time_end}", - query_timestamps=[t_dss], - ) - with self.check("ISA version format", [self._dss.participant_id]) as check: - if not all( - c not in "\0\t\r\n#%/:?@[\]" for c in isa_change.dss_query.isa.version - ): - check.record_failed( - "PUT ISA returned ISA with invalid version format", - Severity.High, - f"DSS returned an ISA with a version that is not URL-safe: {isa_change.dss_query.isa.version}", - query_timestamps=[t_dss], - ) + self.end_test_step() - with self.check("ISA response format", [self._dss.participant_id]) as check: - errors = schema_validation.validate( - self._dss.rid_version.openapi_path, - self._dss.rid_version.openapi_put_isa_response_path, - isa_change.dss_query.query.response.json, - ) - if errors: - details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) - check.record_failed( - "PUT ISA response format was invalid", - Severity.Medium, - "Found the following schema validation errors in the DSS response:\n" - + details, - query_timestamps=[t_dss], - ) + _create_isa_step() - # TODO: Validate subscriber notifications + self._get_isa_by_id_step() - self.end_test_step() + self.end_test_case() def _update_and_search_isa_case(self): self.begin_test_case("Update and search ISA") diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py index 1c89883739..fda32ab3e3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py @@ -81,7 +81,7 @@ def _clean_any_sub(self): "Successful subscription query", [self._dss.participant_id] ) as check: fetched = self._dss_wrapper.search_subs( - check, self._isa.footprint.to_vertices() + check, [vertex.as_s2sphere() for vertex in self._isa.footprint] ) for sub_id in fetched.subscriptions.keys(): with self.check( @@ -232,7 +232,7 @@ def _check_properly_truncated( def _default_subscription_params(self, duration: datetime.timedelta) -> Dict: now = datetime.datetime.utcnow() return dict( - area_vertices=self._isa.footprint.to_vertices(), + area_vertices=[vertex.as_s2sphere() for vertex in self._isa.footprint], alt_lo=self._isa.altitude_min, alt_hi=self._isa.altitude_max, start_time=now, diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py index 38ad78a2b4..e75af70b06 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py @@ -27,14 +27,14 @@ def _default_params(duration: datetime.timedelta) -> Dict: - now = datetime.datetime.utcnow() + now = datetime.datetime.now().astimezone() return dict( area_vertices=VERTICES, alt_lo=20, alt_hi=400, start_time=now, end_time=now + duration, - uss_base_url="https://example.com/uss/flights", + uss_base_url="https://example.com", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 8b9cfea7dd..b797ee037d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -9,6 +9,7 @@ FetchedSubscriptions, RIDQuery, FetchedISA, + FetchedISAs, ) from monitoring.monitorlib.mutate import rid as mutate from monitoring.monitorlib.fetch import rid as fetch @@ -20,6 +21,8 @@ TestScenario, ) +MAX_SKEW = 1e-6 # seconds maximum difference between expected and actual timestamps + class DSSWrapper(object): """Wraps a DSS instance with test checks.""" @@ -165,39 +168,79 @@ def put_isa( self._handle_query_result( check, mutated_isa.dss_query, f"Failed to insert ISA {isa_id}" ) + for notification_query in mutated_isa.notifications.values(): + self._scenario.record_query(notification_query.query) - dt = abs( - mutated_isa.dss_query.isa.time_end.timestamp() - end_time.timestamp() - ) - if dt > 0.001: + t_dss = mutated_isa.dss_query.query.request.timestamp + dss_isa = mutated_isa.dss_query.isa + + if mutated_isa.dss_query.query.status_code == 201: check.record_failed( - summary=f"DSS did not correctly create or update ISA; mismatched end time", - severity=Severity.Medium, + summary=f"PUT ISA returned technically-incorrect 201", + severity=Severity.Low, participants=[self._dss.participant_id], - details=f"Expected: '{end_time}', received: '{mutated_isa.dss_query.isa.time_end}'", - query_timestamps=[mutated_isa.dss_query.query.request.timestamp], + details="DSS should return 200 from PUT ISA, but instead returned the reasonable-but-technically-incorrect code 201", + query_timestamps=[t_dss], ) - elif isa_id != mutated_isa.dss_query.isa.id: + + if isa_id != dss_isa.id: check.record_failed( summary=f"DSS did not return correct ISA", severity=Severity.High, participants=[self._dss.participant_id], - details=f"Expected ISA ID {isa_id} but got {mutated_isa.dss_query.isa.id}", - query_timestamps=[mutated_isa.dss_query.query.request.timestamp], + details=f"Expected ISA ID {isa_id} but got {dss_isa.id}", + query_timestamps=[t_dss], ) - elif ( - isa_version is not None - and mutated_isa.dss_query.isa.version == isa_version - ): + + if isa_version is not None: + if dss_isa.version == isa_version: + check.record_failed( + summary=f"ISA was not modified", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"Got old version {isa_version} while expecting new version", + query_timestamps=[t_dss], + ) + if not all(c not in "\0\t\r\n#%/:?@[\]" for c in dss_isa.version): + check.record_failed( + summary=f"DSS returned ISA (ID {isa_id}) with invalid version format", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"DSS returned an ISA with a version that is not URL-safe: {dss_isa.version}", + query_timestamps=[t_dss], + ) + + if abs((dss_isa.time_start - start_time).total_seconds()) > MAX_SKEW: check.record_failed( - summary=f"ISA was not modified", + summary=f"DSS returned ISA (ID {isa_id}) with incorrect start time", severity=Severity.High, participants=[self._dss.participant_id], - details=f"Got old version {isa_version} while expecting new version", - query_timestamps=[mutated_isa.dss_query.query.request.timestamp], + details=f"DSS should have returned an ISA with a start time of {start_time}, but instead the ISA returned had a start time of {dss_isa.time_start}", + query_timestamps=[t_dss], ) - else: - return mutated_isa + if abs((dss_isa.time_end - end_time).total_seconds()) > MAX_SKEW: + check.record_failed( + summary=f"DSS returned ISA (ID {isa_id}) with incorrect end time", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"DSS should have returned an ISA with an end time of {end_time}, but instead the ISA returned had an end time of {dss_isa.time_end}", + query_timestamps=[t_dss], + ) + + expected_flights_url = self._dss.rid_version.flights_url_of(uss_base_url) + actual_flights_url = dss_isa.flights_url + if actual_flights_url != expected_flights_url: + check.record_failed( + summary=f"DSS returned ISA (ID {isa_id}) with incorrect URL", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"DSS should have returned an ISA with a flights URL of {expected_flights_url}, but instead the ISA returned had a flights URL of {actual_flights_url}", + query_timestamps=[t_dss], + ) + + # TODO: Validate subscriber notifications + + return mutated_isa except QueryError as e: self._handle_query_error(check, e) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index f8b0b8d157..7a7d856e5b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -42,36 +42,39 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub ### Create ISA test step -This step attempts to create an ISA with a 60-minute expiration. +This step attempts to create at the DSS the ISA provided as resource. #### ISA created check If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. -#### ISA ID matches check - When the ISA is created, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. -#### ISA URL matches check - When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -#### ISA start time matches check - The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -#### ISA end time matches check - The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -#### ISA version format check - Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. -#### ISA response format check - The API for **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. +### Get ISA by ID test step + +This step attempts to retrieve at the DSS the ISA just created. + +#### Successful ISA query check + +If the ISA cannot be queried, the GET ISA DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +#### ISA version match check + +The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + + ## Update and search ISA test case ## Delete ISA test case From a8b1a85052d34c2ff374cba9a0914e002b8a3b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Thu, 14 Sep 2023 11:38:03 +0200 Subject: [PATCH 2/9] add v19 ISASimple --- .../scenarios/astm/netrid/v19/dss/__init__.py | 1 + .../astm/netrid/v19/dss/isa_simple.md | 96 +++++++++++++++++++ .../astm/netrid/v19/dss/isa_simple.py | 8 ++ .../suites/astm/netrid/f3411_19.md | 11 ++- .../astm/netrid/f3411_19/dss_probing.md | 17 +++- .../astm/netrid/f3411_19/dss_probing.yaml | 6 ++ 6 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.py diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py index 266bb9c958..3cd77afc8e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py @@ -1 +1,2 @@ +from .isa_simple import ISASimple from .subscription_validation import SubscriptionValidation diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md new file mode 100644 index 0000000000..0d2dde0c15 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -0,0 +1,96 @@ +# ASTM NetRID DSS: Simple ISA test scenario + +## Overview + +Perform basic operations on a single DSS instance to create an ISA and query it during its time of applicability and +after its time of applicability. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing an ISA to be created. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates an ISA with a known ID. This step ensures that ISA does not exist before the start of the main +part of the test. + +#### Successful ISA query check + +**[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0710](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. + +## Create and check ISA test case + +### Create ISA test step + +This step attempts to create at the DSS the ISA provided as resource. + +#### ISA created check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +When the ISA is created, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +The API for **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +### Get ISA by ID test step + +This step attempts to retrieve at the DSS the ISA just created. + +#### Successful ISA query check + +If the ISA cannot be queried, the GET ISA DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +#### ISA version match check + +The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + + +## Update and search ISA test case + +## Delete ISA test case + +## Cleanup + +The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. + +### Successful ISA query check + +**[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +### Removed pre-existing ISA check + +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +### Notified subscriber check + +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.py new file mode 100644 index 0000000000..05ea63b606 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.isa_simple import ( + ISASimple as CommonISASimple, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class ISASimple(TestScenario, CommonISASimple): + pass diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 089ea93df4..4dcc6d98b2 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 A2-6-1,1a Implemented ASTM F3411-19 NetRID DSS interoperability @@ -94,7 +94,7 @@ DSS0030 Implemented - ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior + ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 @@ -234,7 +234,12 @@ NET0710 Implemented - ASTM NetRID nominal behavior + ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior + + + NET0730 + Implemented + ASTM NetRID DSS: Simple ISA interuss
.automated_testing
.rid
.injection diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index 2bf986cf77..6ebc124313 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -4,7 +4,8 @@ ## Actions -1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) +1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v19/dss/isa_simple.md) ([`scenarios.astm.netrid.v19.dss.ISASimple`](../../../../scenarios/astm/netrid/v19/dss/isa_simple.py)) +2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) ## Checked requirements @@ -16,10 +17,10 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 DSS0030 Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -31,4 +32,14 @@ Implemented ASTM NetRID DSS: Subscription Validation + + NET0710 + Implemented + ASTM NetRID DSS: Simple ISA + + + NET0730 + Implemented + ASTM NetRID DSS: Simple ISA + diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml index 13b165753f..8ace89a5e6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml @@ -4,6 +4,12 @@ resources: id_generator: resources.interuss.IDGeneratorResource isa: resources.netrid.ServiceAreaResource actions: + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.dss.ISASimple + resources: + dss: dss + id_generator: id_generator + isa: isa - test_scenario: scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionValidation resources: From 295d5924e436e7836694c828c2ccfe64dbb7ddc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Fri, 15 Sep 2023 17:44:05 +0200 Subject: [PATCH 3/9] [uss_qualifier/scenarios/documentation] Allow additional checks when a step is linked --- .../scenarios/documentation/parsing.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index a90227a53c..f458d3b66a 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -135,17 +135,16 @@ def _parse_test_step( name = name[0 : -len(TEST_STEP_SUFFIX)] url = _url_of(doc_filename + anchors[values[0]]) + checks: List[TestCheckDocumentation] = [] if values[0].children and isinstance( values[0].children[0], marko.block.inline.Link ): - # We should include the content of the linked test step document rather - # than extracting content from this section. - step = _get_linked_test_step(values[0].children[0].dest, doc_filename) - step = TestStepDocumentation(step) - step.name = name - return step + # We include the content of the linked test step document before + # extracting content from this section. + linked_step = _get_linked_test_step(values[0].children[0].dest, doc_filename) + url = linked_step.url + checks = linked_step.checks.copy() - checks: List[TestCheckDocumentation] = [] c = 1 while c < len(values): if isinstance(values[c], marko.block.Heading): From cbf1ac5a8c17434441bcafe6bd2b51a50c0f7952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Fri, 15 Sep 2023 18:10:57 +0200 Subject: [PATCH 4/9] move back checks as part of DSSWrapper --- .../astm/netrid/common/dss/isa_simple.py | 25 +++-- .../netrid/common/dss_interoperability.py | 12 ++- .../scenarios/astm/netrid/dss_wrapper.py | 95 +++++++++++-------- .../astm/netrid/v19/dss/isa_simple.md | 18 +--- .../astm/netrid/v19/dss/test_steps/put_isa.md | 34 +++++++ .../astm/netrid/v19/dss_interoperability.md | 8 +- .../astm/netrid/v22a/dss/isa_simple.md | 18 +--- .../netrid/v22a/dss/test_steps/put_isa.md | 34 +++++++ .../astm/netrid/v22a/dss_interoperability.md | 8 +- .../suites/astm/netrid/f3411_19.md | 2 +- .../suites/astm/netrid/f3411_22a.md | 2 +- .../suites/uspace/network_identification.md | 2 +- .../suites/uspace/required_services.md | 2 +- 13 files changed, 156 insertions(+), 104 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index aacc5051e2..96382fa43d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -141,20 +141,17 @@ def _create_and_check_isa_case(self): def _create_isa_step(): self.begin_test_step("Create ISA") - with self.check("ISA created", [self._dss.participant_id]) as check: - isa_change = self._dss_wrapper.put_isa( - check, - area_vertices=self._isa_area, - start_time=self._isa_start_time, - end_time=self._isa_end_time, - uss_base_url=self._isa.base_url, - isa_id=self._isa_id, - isa_version=self._isa_version, - alt_lo=self._isa.altitude_min, - alt_hi=self._isa.altitude_max, - server_id=self._dss.participant_id, # TODO: check this - ) - self._isa_version = isa_change.dss_query.isa.version + isa_change = self._dss_wrapper.put_isa( + area_vertices=self._isa_area, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + ) + self._isa_version = isa_change.dss_query.isa.version self.end_test_step() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py index e75af70b06..523369be95 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py @@ -143,12 +143,13 @@ def step1(self): isa_1 = self._new_isa("isa_1") mutated_isa = self._dss_primary.put_isa( - check, isa_id=isa_1.uuid, **_default_params(datetime.timedelta(minutes=10)), ) isa_1.version = mutated_isa.dss_query.isa.version + check.record_passed() + def step2(self): """Can create Subscription in all DSSs, ISA accessible from all non-primary DSSs.""" @@ -434,13 +435,14 @@ def step5(self): "Can modify ISA in primary DSS", [self._dss_primary.participant_id] ) as check: mutated_isa = self._dss_primary.put_isa( - check, isa_id=isa_1.uuid, isa_version=isa_1.version, **_default_params(datetime.timedelta(seconds=SHORT_WAIT_SEC)), ) isa_1.version = mutated_isa.dss_query.isa.version + check.record_passed() + # TODO: Implement "ISA modification triggers subscription notification requests check" def step6(self): @@ -543,12 +545,13 @@ def step10(self): "ISA[P] created with proper response", [self._dss_primary.participant_id] ) as check: mutated_isa = self._dss_primary.put_isa( - check, isa_id=isa_2.uuid, **_default_params(datetime.timedelta(minutes=10)), ) isa_2.version = mutated_isa.dss_query.isa.version + check.record_passed() + with self.check( "All Subscription[i] 1≤i≤n returned in subscribers", [self._dss_primary.participant_id], @@ -604,12 +607,13 @@ def step12(self): "ISA[P] created with proper response", [self._dss_primary.participant_id] ) as check: mutated_isa = self._dss_primary.put_isa( - check, isa_id=isa_3.uuid, **_default_params(datetime.timedelta(minutes=10)), ) isa_3.version = mutated_isa.dss_query.isa.version + check.record_passed() + with self.check( "None of Subscription[i] 1≤i≤n returned in subscribers", [self._dss_primary.participant_id], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 9db4f46d75..44cbeb067b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -27,6 +27,8 @@ class DSSWrapper(object): """Wraps a DSS instance with test checks.""" + # TODO: embed checks in all functions (like it is done for put_isa) instead of passing an existing check as parameter + _scenario: TestScenario _dss: DSSInstance @@ -50,6 +52,7 @@ def base_url(self) -> str: def has_private_address(self) -> bool: return self._dss.has_private_address + # TODO: QueryError is not actually raised for RID functions, this function and its uses should be removed def _handle_query_error( self, check: PendingCheck, @@ -134,7 +137,6 @@ def get_isa( def put_isa( self, - check: PendingCheck, area_vertices: List[s2sphere.LatLng], alt_lo: float, alt_hi: float, @@ -145,36 +147,32 @@ def put_isa( isa_version: Optional[str] = None, ) -> ISAChange: """Create or update an ISA at the DSS. - A check fail is considered of high severity and as such will raise a ScenarioCannotContinueError. - Fails if the ID of the ISA returned by the DSS does not match the submitted one. - Fails if the end time of the ISA returned by the DSS does not match the submitted one. + Most check fail are considered of high severity and as such will raise a ScenarioCannotContinueError. + This function implements the test step described in '[v19|v22a]/dss/test_steps/put_isa.md'. :return: the DSS response """ - try: - mutated_isa = mutate.put_isa( - area_vertices=area_vertices, - alt_lo=alt_lo, - alt_hi=alt_hi, - start_time=start_time, - end_time=end_time, - uss_base_url=uss_base_url, - isa_id=isa_id, - isa_version=isa_version, - rid_version=self._dss.rid_version, - utm_client=self._dss.client, - server_id=self._dss.participant_id, - ) + mutated_isa = mutate.put_isa( + area_vertices=area_vertices, + alt_lo=alt_lo, + alt_hi=alt_hi, + start_time=start_time, + end_time=end_time, + uss_base_url=uss_base_url, + isa_id=isa_id, + isa_version=isa_version, + rid_version=self._dss.rid_version, + utm_client=self._dss.client, + server_id=self._dss.participant_id, + ) + t_dss = mutated_isa.dss_query.query.request.timestamp + dss_isa = mutated_isa.dss_query.isa + with self._scenario.check("ISA created", [self._dss.participant_id]) as check: self._handle_query_result( check, mutated_isa.dss_query, f"Failed to insert ISA {isa_id}" ) - for notification_query in mutated_isa.notifications.values(): - self._scenario.record_query(notification_query.query) - - t_dss = mutated_isa.dss_query.query.request.timestamp - dss_isa = mutated_isa.dss_query.isa if mutated_isa.dss_query.query.status_code == 201: check.record_failed( @@ -185,6 +183,12 @@ def put_isa( query_timestamps=[t_dss], ) + for notification_query in mutated_isa.notifications.values(): + self._scenario.record_query(notification_query.query) + + with self._scenario.check( + "ISA ID matches", [self._dss.participant_id] + ) as check: if isa_id != dss_isa.id: check.record_failed( summary=f"DSS did not return correct ISA", @@ -194,7 +198,10 @@ def put_isa( query_timestamps=[t_dss], ) - if isa_version is not None: + if isa_version is not None: + with self._scenario.check( + "ISA version changed", [self._dss.participant_id] + ) as check: if dss_isa.version == isa_version: check.record_failed( summary=f"ISA was not modified", @@ -203,15 +210,22 @@ def put_isa( details=f"Got old version {isa_version} while expecting new version", query_timestamps=[t_dss], ) - if not all(c not in "\0\t\r\n#%/:?@[\]" for c in dss_isa.version): - check.record_failed( - summary=f"DSS returned ISA (ID {isa_id}) with invalid version format", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"DSS returned an ISA with a version that is not URL-safe: {dss_isa.version}", - query_timestamps=[t_dss], - ) + with self._scenario.check( + "ISA version format", [self._dss.participant_id] + ) as check: + if not all(c not in "\0\t\r\n#%/:?@[\]" for c in dss_isa.version): + check.record_failed( + summary=f"DSS returned ISA (ID {isa_id}) with invalid version format", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"DSS returned an ISA with a version that is not URL-safe: {dss_isa.version}", + query_timestamps=[t_dss], + ) + + with self._scenario.check( + "ISA start time matches", [self._dss.participant_id] + ) as check: if abs((dss_isa.time_start - start_time).total_seconds()) > MAX_SKEW: check.record_failed( summary=f"DSS returned ISA (ID {isa_id}) with incorrect start time", @@ -220,6 +234,10 @@ def put_isa( details=f"DSS should have returned an ISA with a start time of {start_time}, but instead the ISA returned had a start time of {dss_isa.time_start}", query_timestamps=[t_dss], ) + + with self._scenario.check( + "ISA end time matches", [self._dss.participant_id] + ) as check: if abs((dss_isa.time_end - end_time).total_seconds()) > MAX_SKEW: check.record_failed( summary=f"DSS returned ISA (ID {isa_id}) with incorrect end time", @@ -229,6 +247,9 @@ def put_isa( query_timestamps=[t_dss], ) + with self._scenario.check( + "ISA URL matches", [self._dss.participant_id] + ) as check: expected_flights_url = self._dss.rid_version.flights_url_of(uss_base_url) actual_flights_url = dss_isa.flights_url if actual_flights_url != expected_flights_url: @@ -240,15 +261,9 @@ def put_isa( query_timestamps=[t_dss], ) - # TODO: Validate subscriber notifications - - return mutated_isa + # TODO: Validate subscriber notifications - except QueryError as e: - self._handle_query_error(check, e) - raise RuntimeError( - "DSS query was not successful, but a High Severity issue didn't interrupt execution" - ) + return mutated_isa def del_isa( self, diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index 0d2dde0c15..1cf6f6b767 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -40,26 +40,10 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub ## Create and check ISA test case -### Create ISA test step +### [Create ISA test step](test_steps/put_isa.md) This step attempts to create at the DSS the ISA provided as resource. -#### ISA created check - -If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. - -When the ISA is created, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. - -When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. - -The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. - -The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. - -Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. - -The API for **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. - ### Get ISA by ID test step This step attempts to retrieve at the DSS the ISA just created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md new file mode 100644 index 0000000000..4e6ca44068 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md @@ -0,0 +1,34 @@ +# Create or update ISA test step + +This page describes the content of a common test step where a creation or an update of an ISA should be successful. +See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). + +## ISA created check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +The API for **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +## ISA ID matches check + +When the ISA is created, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +## ISA version changed check + +When the ISA is updated, the DSS returns the updated version of the ISA in the response body. If this version remains the same as the one before the update, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +## ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +## ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +## ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +## ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md index f822ff8e74..4207b79a8d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md @@ -52,7 +52,7 @@ As such, this check will fail if the DSS is not reachable with a dummy query, ## Interoperability sequence test case -### S1 test step +### [S1 test step](dss/test_steps/put_isa.md) Action: USS1@DSS*P*: PUT ISA with no start time and end time 10 minutes from now @@ -136,7 +136,7 @@ Qualitatively proves: All Subscription[i] 1≤i≤n are returned in subscription **[astm.f3411.v19.DSS0070](../../../../requirements/astm/f3411/v19.md)** requires that all DSS instances in a pool return the same result. This check fails if the DSS instance does not return the same result as the other DSS instances. -### S5 test step +### [S5 test step](dss/test_steps/put_isa.md) Action: USS1@DSS*P*: PUT ISA[*P*] setting end time to now + D seconds @@ -210,7 +210,7 @@ Qualitatively proves: Expired ISA automatically removed, ISA modifications acces **[astm.f3411.v19.DSS0070](../../../../requirements/astm/f3411/v19.md)** requires that all DSS instances in a pool return the same result. This check fails if the DSS instance does not return the same result as the other DSS instances. -### S10 test step +### [S10 test step](dss/test_steps/put_isa.md) Action: USS1@DSS*P*: PUT ISA with no start time and end time 10 minutes from now @@ -238,7 +238,7 @@ Qualitatively proves: ISA deletion triggers subscription notification requests **[astm.f3411.v19.A2-6-1,3c](../../../../requirements/astm/f3411/v19.md)** -### S12 test step +### [S12 test step](dss/test_steps/put_isa.md) Action: Wait >D seconds from S9 then USS1@DSS*P*: PUT ISA with no start time and end time 10 minutes from now diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 7a7d856e5b..51fd0b3c21 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -40,26 +40,10 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub ## Create and check ISA test case -### Create ISA test step +### [Create ISA test step](test_steps/put_isa.md) This step attempts to create at the DSS the ISA provided as resource. -#### ISA created check - -If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. - -When the ISA is created, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. - -When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. - -The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. - -The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. - -Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. - -The API for **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. - ### Get ISA by ID test step This step attempts to retrieve at the DSS the ISA just created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md new file mode 100644 index 0000000000..07e31b84ab --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md @@ -0,0 +1,34 @@ +# Create or update ISA test step + +This page describes the content of a common test step where a creation or an update of an ISA should be successful. +See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). + +## ISA created check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +The API for **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +## ISA ID matches check + +When the ISA is created, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +## ISA version changed check + +When the ISA is updated, the DSS returns the updated version of the ISA in the response body. If this version remains the same as the one before the update, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +## ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +## ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +## ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +## ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md index 85898a4f18..7d67f67959 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md @@ -51,7 +51,7 @@ As such, this check will fail if the DSS is not reachable with a dummy query, ## Interoperability sequence test case -### S1 test step +### [S1 test step](dss/test_steps/put_isa.md) Action: USS1@DSS*P*: PUT ISA with no start time and end time 10 minutes from now @@ -135,7 +135,7 @@ Qualitatively proves: All Subscription[i] 1≤i≤n are returned in subscription **[astm.f3411.v22a.DSS0070](../../../../requirements/astm/f3411/v22a.md)** requires that all DSS instances in a pool return the same result. This check fails if the DSS instance does not return the same result as the other DSS instances. -### S5 test step +### [S5 test step](dss/test_steps/put_isa.md) Action: USS1@DSS*P*: PUT ISA[*P*] setting end time to now + D seconds @@ -209,7 +209,7 @@ Qualitatively proves: Expired ISA automatically removed, ISA modifications acces **[astm.f3411.v22a.DSS0070](../../../../requirements/astm/f3411/v22a.md)** requires that all DSS instances in a pool return the same result. This check fails if the DSS instance does not return the same result as the other DSS instances. -### S10 test step +### [S10 test step](dss/test_steps/put_isa.md) Action: USS1@DSS*P*: PUT ISA with no start time and end time 10 minutes from now @@ -237,7 +237,7 @@ Qualitatively proves: ISA deletion triggers subscription notification requests **[astm.f3411.v22a.A2-6-1,3c](../../../../requirements/astm/f3411/v22a.md)** -### S12 test step +### [S12 test step](dss/test_steps/put_isa.md) Action: Wait >D seconds from S9 then USS1@DSS*P*: PUT ISA with no start time and end time 10 minutes from now diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 57805c07aa..8a47fe14d6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -96,7 +96,7 @@ DSS0030 Implemented - ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 2afb0a95bf..9ec473ac45 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -96,7 +96,7 @@ DSS0030 Implemented - ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index b4e6c18acb..b1359ca95b 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -89,7 +89,7 @@ DSS0030 Implemented - ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 22f2424919..1aec52ae33 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -90,7 +90,7 @@ DSS0030 Implemented - ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 From ad8a00d5f5957ff8a53746a28ed64697295a9393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Fri, 15 Sep 2023 18:35:48 +0200 Subject: [PATCH 5/9] fixes following review --- monitoring/monitorlib/geo.py | 21 +++++++------------ monitoring/monitorlib/mutate/rid.py | 6 +++--- .../resources/netrid/service_area.py | 16 ++++++++------ .../astm/netrid/v19/dss/isa_simple.md | 8 +++---- .../astm/netrid/v22a/dss/isa_simple.md | 8 +++---- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/monitoring/monitorlib/geo.py b/monitoring/monitorlib/geo.py index f921552bc0..82d18b12ed 100644 --- a/monitoring/monitorlib/geo.py +++ b/monitoring/monitorlib/geo.py @@ -19,8 +19,16 @@ class DistanceUnits(str, Enum): class LatLngPoint(ImplicitDict): + """Vertex in latitude and longitude""" + lat: float + """Latitude (degrees)""" + lng: float + """Longitude (degrees)""" + + def as_s2sphere(self) -> s2sphere.LatLng: + return s2sphere.LatLng.from_degrees(self.lat, self.lng) class Radius(ImplicitDict): @@ -251,16 +259,3 @@ def to_vertices(self) -> List[s2sphere.LatLng]: s2sphere.LatLng.from_degrees(self.lat_max, self.lng_max), s2sphere.LatLng.from_degrees(self.lat_min, self.lng_max), ] - - -class LatLngVertex(ImplicitDict): - """Vertex in latitude and longitude""" - - lat: float - """Latitude (degrees)""" - - lng: float - """Longitude (degrees)""" - - def as_s2sphere(self) -> s2sphere.LatLng: - return s2sphere.LatLng.from_degrees(self.lat, self.lng) diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index 2022a0b01e..7a08f87149 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -373,11 +373,11 @@ def errors(self) -> List[str]: # The augmented.yaml version of the definition should be fixed and then this can be applied to both version. if self.rid_version == RIDVersion.f3411_22a: validation_errors = schema_validation.validate( - self.rid_version.openapi_path, - self.rid_version.openapi_delete_isa_response_path + openapi_path=self.rid_version.openapi_path, + object_path=self.rid_version.openapi_delete_isa_response_path if self.mutation == "delete" else self.rid_version.openapi_put_isa_response_path, - self.query.response.json, + instance=self.query.response.json, ) if validation_errors: return [ diff --git a/monitoring/uss_qualifier/resources/netrid/service_area.py b/monitoring/uss_qualifier/resources/netrid/service_area.py index 9e2e01224a..826007354e 100644 --- a/monitoring/uss_qualifier/resources/netrid/service_area.py +++ b/monitoring/uss_qualifier/resources/netrid/service_area.py @@ -2,7 +2,7 @@ from typing import List from implicitdict import ImplicitDict, StringBasedDateTime -from monitoring.monitorlib.geo import LatLngVertex +from monitoring.monitorlib.geo import LatLngPoint from monitoring.uss_qualifier.resources.resource import Resource @@ -15,7 +15,7 @@ class ServiceAreaSpecification(ImplicitDict): This URL will probably not identify a real resource in tests.""" - footprint: List[LatLngVertex] + footprint: List[LatLngPoint] """2D outline of service area""" altitude_min: float = 0 @@ -33,12 +33,16 @@ class ServiceAreaSpecification(ImplicitDict): time_end: StringBasedDateTime """End time of service area (relative to reference_time)""" - def shifted_time_start(self, now: datetime.datetime) -> datetime.datetime: - dt = now - self.reference_time.datetime + def shifted_time_start( + self, new_reference_time: datetime.datetime + ) -> datetime.datetime: + dt = new_reference_time - self.reference_time.datetime return self.time_start.datetime + dt - def shifted_time_end(self, now: datetime.datetime) -> datetime.datetime: - dt = now - self.reference_time.datetime + def shifted_time_end( + self, new_reference_time: datetime.datetime + ) -> datetime.datetime: + dt = new_reference_time - self.reference_time.datetime return self.time_end.datetime + dt diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index 1cf6f6b767..348b72f318 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -42,11 +42,11 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub ### [Create ISA test step](test_steps/put_isa.md) -This step attempts to create at the DSS the ISA provided as resource. +This step attempts to query the configured DSS with the ISA provided as a resource. ### Get ISA by ID test step -This step attempts to retrieve at the DSS the ISA just created. +This step attempts to retrieve the previously created ISA from the DSS. #### Successful ISA query check @@ -73,8 +73,8 @@ The cleanup phase of this test scenario attempts to remove the ISA if the test e ### Removed pre-existing ISA check -If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. ### Notified subscriber check -When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 51fd0b3c21..79c10a6413 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -42,11 +42,11 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub ### [Create ISA test step](test_steps/put_isa.md) -This step attempts to create at the DSS the ISA provided as resource. +This step attempts to query the configured DSS with the ISA provided as a resource. ### Get ISA by ID test step -This step attempts to retrieve at the DSS the ISA just created. +This step attempts to retrieve the previously created ISA from the DSS. #### Successful ISA query check @@ -73,8 +73,8 @@ The cleanup phase of this test scenario attempts to remove the ISA if the test e ### Removed pre-existing ISA check -If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. ### Notified subscriber check -When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. From 16f054e54caf5ad65ae09e63bc91878130945b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Wed, 20 Sep 2023 16:47:13 +0200 Subject: [PATCH 6/9] update submodules and uas_standards --- interfaces/rid/v1 | 2 +- monitoring/monitorlib/mutate/rid.py | 28 ++++++++----------- .../suites/astm/netrid/f3411_19.md | 2 +- requirements.txt | 2 +- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/interfaces/rid/v1 b/interfaces/rid/v1 index 935f3b64cf..228b301be1 160000 --- a/interfaces/rid/v1 +++ b/interfaces/rid/v1 @@ -1 +1 @@ -Subproject commit 935f3b64cf252e82777a74037001e17420bc3803 +Subproject commit 228b301be1f4466df9b014f20b515665861a8e67 diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index 7a08f87149..e7f9773535 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -368,22 +368,18 @@ def errors(self) -> List[str]: f"Error parsing F3411-22a USS PutIdentificationServiceAreaResponse: {str(e)}" ] - # TODO: This validates the schema only for v22a as the v19 OpenAPI definition contains a mistake in - # 'components.schemas.SubscriptionState', the non-existing property 'subscription' is marked as required. - # The augmented.yaml version of the definition should be fixed and then this can be applied to both version. - if self.rid_version == RIDVersion.f3411_22a: - validation_errors = schema_validation.validate( - openapi_path=self.rid_version.openapi_path, - object_path=self.rid_version.openapi_delete_isa_response_path - if self.mutation == "delete" - else self.rid_version.openapi_put_isa_response_path, - instance=self.query.response.json, - ) - if validation_errors: - return [ - f"PUT ISA response JSON validation error: [{e.json_path}] {e.message}" - for e in validation_errors - ] + validation_errors = schema_validation.validate( + openapi_path=self.rid_version.openapi_path, + object_path=self.rid_version.openapi_delete_isa_response_path + if self.mutation == "delete" + else self.rid_version.openapi_put_isa_response_path, + instance=self.query.response.json, + ) + if validation_errors: + return [ + f"PUT ISA response JSON validation error: [{e.json_path}] {e.message}" + for e in validation_errors + ] return [] diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 68ad39e47f..f6a382650f 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -23,7 +23,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 A2-6-1,1a Implemented ASTM F3411-19 NetRID DSS interoperability diff --git a/requirements.txt b/requirements.txt index a4c773ff81..cee4d53f0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,5 +39,5 @@ s2sphere==0.2.5 shapely==1.7.1 structlog==21.5.0 # deployment_manager termcolor==1.1.0 -uas_standards==2.0.0 +uas_standards==2.1.0 Werkzeug==2.0.3 # See https://github.com/interuss/dss/issues/753 From 1091de6592b1955a0468af35a1788ac014055c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Wed, 20 Sep 2023 17:24:23 +0200 Subject: [PATCH 7/9] [uss_qualifier/scenarios/aggregate_checks] Convert unattributed queries check to a note --- .../astm/netrid/common/aggregate_checks.py | 15 ++++----------- .../scenarios/astm/netrid/v19/aggregate_checks.md | 7 ------- .../astm/netrid/v22a/aggregate_checks.md | 7 ------- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index ade57069a2..9edd3b416e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -154,17 +154,10 @@ def _verify_https_everywhere(self): if query.get("server_id") is None ] if len(unattr_queries) > 0: - # TODO clean this up: this is an internal requirement and not a check, - # leaving as-is during development to make sure the test-suite runs but we know about unattributed queries - # ultimately this check could go into the constructor and blow things up early - - with self.check( - "No unattributed queries", - [], - ) as check: - check.record_failed( - f"found unattributed queries: {unattr_queries}", Severity.Medium - ) + self.record_note( + "unattributed-queries", + f"found unattributed queries: {unattr_queries}", + ) def _inspect_participant_queries( self, participant_id: str, participant_queries: List[fetch.Query] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 5c945cbfd7..ada4d2df92 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -67,13 +67,6 @@ as being in "local debug" mode, they may serve requests over http without produc If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. -#### No unattributed queries check - -All queries must have been attributed to a participant: an unattributed query means that one of the test cases has not -properly recorded to which participant it was made. - -This is an internal requirement and does not necessarily imply that there is a problem with the participants under test. - ## Mock USS interactions evaluation test case In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index a7a4808f24..d5e721678e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -63,13 +63,6 @@ of the durations for the replies to requested flights in an area do not exceed t Inspects all record queries for their usage of https. If services such as a service provider, observer or DSS are marked as being in "local debug" mode, they may serve requests over http without producing failed checks despite their lack of encryption. -#### No unattributed queries check - -All queries must have been attributed to a participant: an unattributed query means that one of the test cases has not -properly recorded to which participant it was made. - -This is an internal requirement and does not necessarily imply that there is a problem with the participants under test. - #### All interactions happen over https check If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v22a.NET0220](../../../../requirements/astm/f3411/v22a.md)** is not satisfied. From 534457b3f1301ecc235590bcb3cc62b76e4cbb9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Thu, 21 Sep 2023 09:03:25 +0200 Subject: [PATCH 8/9] DSS interop: put back record fail in check --- .../astm/netrid/common/dss/isa_simple.py | 24 +-- .../netrid/common/dss_interoperability.py | 12 +- .../scenarios/astm/netrid/dss_wrapper.py | 144 ++++++++++-------- .../astm/netrid/v19/dss/isa_simple.md | 4 + .../astm/netrid/v19/dss/test_steps/put_isa.md | 6 +- .../astm/netrid/v22a/dss/isa_simple.md | 4 + .../netrid/v22a/dss/test_steps/put_isa.md | 6 +- 7 files changed, 112 insertions(+), 88 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index 96382fa43d..06aff7fe90 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -141,17 +141,19 @@ def _create_and_check_isa_case(self): def _create_isa_step(): self.begin_test_step("Create ISA") - isa_change = self._dss_wrapper.put_isa( - area_vertices=self._isa_area, - start_time=self._isa_start_time, - end_time=self._isa_end_time, - uss_base_url=self._isa.base_url, - isa_id=self._isa_id, - isa_version=self._isa_version, - alt_lo=self._isa.altitude_min, - alt_hi=self._isa.altitude_max, - ) - self._isa_version = isa_change.dss_query.isa.version + with self.check("ISA created", [self._dss.participant_id]) as check: + isa_change = self._dss_wrapper.put_isa( + main_check=check, + area_vertices=self._isa_area, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + ) + self._isa_version = isa_change.dss_query.isa.version self.end_test_step() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py index 523369be95..e75af70b06 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py @@ -143,13 +143,12 @@ def step1(self): isa_1 = self._new_isa("isa_1") mutated_isa = self._dss_primary.put_isa( + check, isa_id=isa_1.uuid, **_default_params(datetime.timedelta(minutes=10)), ) isa_1.version = mutated_isa.dss_query.isa.version - check.record_passed() - def step2(self): """Can create Subscription in all DSSs, ISA accessible from all non-primary DSSs.""" @@ -435,14 +434,13 @@ def step5(self): "Can modify ISA in primary DSS", [self._dss_primary.participant_id] ) as check: mutated_isa = self._dss_primary.put_isa( + check, isa_id=isa_1.uuid, isa_version=isa_1.version, **_default_params(datetime.timedelta(seconds=SHORT_WAIT_SEC)), ) isa_1.version = mutated_isa.dss_query.isa.version - check.record_passed() - # TODO: Implement "ISA modification triggers subscription notification requests check" def step6(self): @@ -545,13 +543,12 @@ def step10(self): "ISA[P] created with proper response", [self._dss_primary.participant_id] ) as check: mutated_isa = self._dss_primary.put_isa( + check, isa_id=isa_2.uuid, **_default_params(datetime.timedelta(minutes=10)), ) isa_2.version = mutated_isa.dss_query.isa.version - check.record_passed() - with self.check( "All Subscription[i] 1≤i≤n returned in subscribers", [self._dss_primary.participant_id], @@ -607,13 +604,12 @@ def step12(self): "ISA[P] created with proper response", [self._dss_primary.participant_id] ) as check: mutated_isa = self._dss_primary.put_isa( + check, isa_id=isa_3.uuid, **_default_params(datetime.timedelta(minutes=10)), ) isa_3.version = mutated_isa.dss_query.isa.version - check.record_passed() - with self.check( "None of Subscription[i] 1≤i≤n returned in subscribers", [self._dss_primary.participant_id], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 44cbeb067b..dec62fe5d7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -3,6 +3,7 @@ from typing import Optional, List, Set +from monitoring.monitorlib import schema_validation from monitoring.monitorlib.fetch import QueryError from monitoring.monitorlib.fetch.rid import ( FetchedSubscription, @@ -137,6 +138,7 @@ def get_isa( def put_isa( self, + main_check: PendingCheck, area_vertices: List[s2sphere.LatLng], alt_lo: float, alt_hi: float, @@ -147,8 +149,9 @@ def put_isa( isa_version: Optional[str] = None, ) -> ISAChange: """Create or update an ISA at the DSS. - Most check fail are considered of high severity and as such will raise a ScenarioCannotContinueError. - This function implements the test step described in '[v19|v22a]/dss/test_steps/put_isa.md'. + + Query failure will fail the provided main check. If the query is successful, the sub-checks of the test step + described in '[v19|v22a]/dss/test_steps/put_isa.md' are performed. Some of those might fail the main check. :return: the DSS response """ @@ -166,99 +169,110 @@ def put_isa( utm_client=self._dss.client, server_id=self._dss.participant_id, ) + self._handle_query_result( + main_check, mutated_isa.dss_query, f"Failed to insert ISA {isa_id}" + ) + for notification_query in mutated_isa.notifications.values(): + self._scenario.record_query(notification_query.query) + + dss_id = [self._dss.participant_id] t_dss = mutated_isa.dss_query.query.request.timestamp dss_isa = mutated_isa.dss_query.isa - with self._scenario.check("ISA created", [self._dss.participant_id]) as check: - self._handle_query_result( - check, mutated_isa.dss_query, f"Failed to insert ISA {isa_id}" - ) - + # sub-checks that do not fail the main check + with self._scenario.check("ISA response code", dss_id) as sub_check: if mutated_isa.dss_query.query.status_code == 201: - check.record_failed( + sub_check.record_failed( summary=f"PUT ISA returned technically-incorrect 201", severity=Severity.Low, - participants=[self._dss.participant_id], details="DSS should return 200 from PUT ISA, but instead returned the reasonable-but-technically-incorrect code 201", query_timestamps=[t_dss], ) - for notification_query in mutated_isa.notifications.values(): - self._scenario.record_query(notification_query.query) + with self._scenario.check("ISA response format", dss_id) as sub_check: + errors = schema_validation.validate( + self._dss.rid_version.openapi_path, + self._dss.rid_version.openapi_put_isa_response_path, + mutated_isa.dss_query.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + sub_check.record_failed( + "PUT ISA response format was invalid", + Severity.Medium, + "Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) + + # sub-checks that fail the main check + def _fail_sub_check( + _sub_check: PendingCheck, _summary: str, _details: str + ) -> None: + """Fails with Medium severity the sub_check and with High severity the main check.""" + + _sub_check.record_failed( + summary=_summary, + severity=Severity.Medium, + details=_details, + query_timestamps=[t_dss], + ) + main_check.record_failed( + summary=f"PUT ISA request succeeded, but the DSS response is not valid: {_summary}", + severity=Severity.High, + details=_details, + query_timestamps=[t_dss], + ) - with self._scenario.check( - "ISA ID matches", [self._dss.participant_id] - ) as check: + with self._scenario.check("ISA ID matches", dss_id) as sub_check: if isa_id != dss_isa.id: - check.record_failed( - summary=f"DSS did not return correct ISA", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"Expected ISA ID {isa_id} but got {dss_isa.id}", - query_timestamps=[t_dss], + _fail_sub_check( + sub_check, + "DSS did not return correct ISA", + f"Expected ISA ID {isa_id} but got {dss_isa.id}", ) if isa_version is not None: - with self._scenario.check( - "ISA version changed", [self._dss.participant_id] - ) as check: + with self._scenario.check("ISA version changed", dss_id) as sub_check: if dss_isa.version == isa_version: - check.record_failed( - summary=f"ISA was not modified", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"Got old version {isa_version} while expecting new version", - query_timestamps=[t_dss], + _fail_sub_check( + sub_check, + "ISA was not modified", + f"Got old version {isa_version} while expecting new version", ) - with self._scenario.check( - "ISA version format", [self._dss.participant_id] - ) as check: + with self._scenario.check("ISA version format", dss_id) as sub_check: if not all(c not in "\0\t\r\n#%/:?@[\]" for c in dss_isa.version): - check.record_failed( - summary=f"DSS returned ISA (ID {isa_id}) with invalid version format", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"DSS returned an ISA with a version that is not URL-safe: {dss_isa.version}", - query_timestamps=[t_dss], + _fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with invalid version format", + f"DSS returned an ISA with a version that is not URL-safe: {dss_isa.version}", ) - with self._scenario.check( - "ISA start time matches", [self._dss.participant_id] - ) as check: + with self._scenario.check("ISA start time matches", dss_id) as sub_check: if abs((dss_isa.time_start - start_time).total_seconds()) > MAX_SKEW: - check.record_failed( - summary=f"DSS returned ISA (ID {isa_id}) with incorrect start time", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"DSS should have returned an ISA with a start time of {start_time}, but instead the ISA returned had a start time of {dss_isa.time_start}", - query_timestamps=[t_dss], + _fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with incorrect start time", + f"DSS should have returned an ISA with a start time of {start_time}, but instead the ISA returned had a start time of {dss_isa.time_start}", ) - with self._scenario.check( - "ISA end time matches", [self._dss.participant_id] - ) as check: + with self._scenario.check("ISA end time matches", dss_id) as sub_check: if abs((dss_isa.time_end - end_time).total_seconds()) > MAX_SKEW: - check.record_failed( - summary=f"DSS returned ISA (ID {isa_id}) with incorrect end time", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"DSS should have returned an ISA with an end time of {end_time}, but instead the ISA returned had an end time of {dss_isa.time_end}", - query_timestamps=[t_dss], + _fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with incorrect end time", + f"DSS should have returned an ISA with an end time of {end_time}, but instead the ISA returned had an end time of {dss_isa.time_end}", ) - with self._scenario.check( - "ISA URL matches", [self._dss.participant_id] - ) as check: + with self._scenario.check("ISA URL matches", dss_id) as sub_check: expected_flights_url = self._dss.rid_version.flights_url_of(uss_base_url) actual_flights_url = dss_isa.flights_url if actual_flights_url != expected_flights_url: - check.record_failed( - summary=f"DSS returned ISA (ID {isa_id}) with incorrect URL", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"DSS should have returned an ISA with a flights URL of {expected_flights_url}, but instead the ISA returned had a flights URL of {actual_flights_url}", - query_timestamps=[t_dss], + _fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with incorrect URL", + f"DSS should have returned an ISA with a flights URL of {expected_flights_url}, but instead the ISA returned had a flights URL of {actual_flights_url}", ) # TODO: Validate subscriber notifications diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index 348b72f318..a1cce5c6d5 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -44,6 +44,10 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub This step attempts to query the configured DSS with the ISA provided as a resource. +#### ISA created check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + ### Get ISA by ID test step This step attempts to retrieve the previously created ISA from the DSS. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md index 4e6ca44068..cddd227af7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md @@ -3,9 +3,11 @@ This page describes the content of a common test step where a creation or an update of an ISA should be successful. See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISA created check +## ISA response code check -If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. +The API for **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** specifies that the code of successful DSS responses is 200. While tolerated in some cases, if the DSS responds with an HTTP code 201 for success, this check will fail with a low severity. + +## ISA response format check The API for **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 5d4b03dc8f..c0fee7ad4b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -44,6 +44,10 @@ When a pre-existing ISA needs to be deleted to ensure a clean workspace, any sub This step attempts to query the configured DSS with the ISA provided as a resource. +#### ISA created check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + ### Get ISA by ID test step This step attempts to retrieve the previously created ISA from the DSS. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md index 07e31b84ab..495959a7d8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md @@ -3,9 +3,11 @@ This page describes the content of a common test step where a creation or an update of an ISA should be successful. See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISA created check +## ISA response code check -If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. +The API for **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** specifies that the code of successful DSS responses is 200. While tolerated in some cases, if the DSS responds with an HTTP code 201 for success, this check will fail with a low severity. + +## ISA response format check The API for **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. From a38e8ba438f3e0ae36aaef8a6264b0e88641e5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Thu, 21 Sep 2023 09:09:41 +0200 Subject: [PATCH 9/9] cleanups --- monitoring/monitorlib/mutate/rid.py | 16 ---------------- monitoring/monitorlib/rid.py | 9 --------- monitoring/monitorlib/schema_validation.py | 6 ------ .../scenarios/astm/netrid/dss_wrapper.py | 2 +- 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index e7f9773535..dd42e2cfc4 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -19,7 +19,6 @@ infrastructure, rid_v1, rid_v2, - schema_validation, ) @@ -67,8 +66,6 @@ def errors(self) -> List[str]: f"Error parsing F3411-22a USS PutSubscriptionResponse: {str(e)}" ] - # TODO: add schema validation (like ChangedISA) - return [] @property @@ -368,19 +365,6 @@ def errors(self) -> List[str]: f"Error parsing F3411-22a USS PutIdentificationServiceAreaResponse: {str(e)}" ] - validation_errors = schema_validation.validate( - openapi_path=self.rid_version.openapi_path, - object_path=self.rid_version.openapi_delete_isa_response_path - if self.mutation == "delete" - else self.rid_version.openapi_put_isa_response_path, - instance=self.query.response.json, - ) - if validation_errors: - return [ - f"PUT ISA response JSON validation error: [{e.json_path}] {e.message}" - for e in validation_errors - ] - return [] @property diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index 22a612b3b8..ce1509035d 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -57,15 +57,6 @@ def openapi_put_isa_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") - @property - def openapi_delete_isa_response_path(self) -> str: - if self == RIDVersion.f3411_19: - return schema_validation.F3411_19.DeleteIdentificationServiceAreaResponse - elif self == RIDVersion.f3411_22a: - return schema_validation.F3411_22a.DeleteIdentificationServiceAreaResponse - else: - raise ValueError(f"Unsupported RID version '{self}'") - @property def realtime_period(self) -> timedelta: if self == RIDVersion.f3411_19: diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index 56ff259f62..29346ed7cb 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -19,9 +19,6 @@ class F3411_19(str, Enum): PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) - DeleteIdentificationServiceAreaResponse = ( - "components.schemas.DeleteIdentificationServiceAreaResponse" - ) class F3411_22a(str, Enum): @@ -31,9 +28,6 @@ class F3411_22a(str, Enum): PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) - DeleteIdentificationServiceAreaResponse = ( - "components.schemas.DeleteIdentificationServiceAreaResponse" - ) class F3548_21(str, Enum): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index dec62fe5d7..32b9dca914 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -28,7 +28,7 @@ class DSSWrapper(object): """Wraps a DSS instance with test checks.""" - # TODO: embed checks in all functions (like it is done for put_isa) instead of passing an existing check as parameter + # TODO: adapt other functions with corresponding test step and sub-checks like it is done for put_isa _scenario: TestScenario _dss: DSSInstance