From ba5d25e6cd6b43b18ad105ee0fb170b06e9e3a38 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Mon, 4 Nov 2024 13:59:24 +0100 Subject: [PATCH] [uss_qualifier] netrid: dss0130 check observed height --- .../astm/netrid/display_data_evaluator.py | 76 +++++++++++++++++++ .../astm/netrid/v19/nominal_behavior.md | 4 + .../astm/netrid/v22a/nominal_behavior.md | 4 + .../suites/astm/netrid/f3411_19.md | 5 ++ .../suites/astm/netrid/f3411_22a.md | 5 ++ .../suites/uspace/network_identification.md | 5 ++ .../suites/uspace/required_services.md | 5 ++ 7 files changed, 104 insertions(+) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index fc9312bf3a..4dd7e54cfc 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -6,6 +6,8 @@ import s2sphere from loguru import logger from s2sphere import LatLng, LatLngRect +from uas_standards.astm.f3411.v22a.api import RIDHeightReference + from uas_standards.interuss.automated_testing.rid.v1.observation import ( Flight, GetDisplayDataResponse, @@ -50,6 +52,9 @@ def _rect_str(rect) -> str: VERTICAL_SPEED_PRECISION = 0.1 SPEED_PRECISION = 0.05 TIMESTAMP_ACCURACY_PRECISION = 0.05 +HEIGHT_PRECISION_M = 1 +# 20km above ground sounds like a reasonable maximum altitude before declaring it as an error +MAX_HEIGHT_M = 20000 @dataclass @@ -1088,6 +1093,77 @@ def _evaluate_normal_sp_observation( details=f"Timestamp accuracy in Service Provider {mapping.injected_flight.uss_participant_id}'s response for flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} with telemetry index {mapping.telemetry_index} is {mapping.observed_flight.timestamp_accuracy} which is not equal to the injected value of {injected_telemetry.timestamp_accuracy}", ) + # height is an optional field. Evaluate only if present: + if "height" in observed_position: + with self._test_scenario.check("Service Provider height") as check: + if injected_position.height is not None: + # Injected data specifies a height, let's compare: + if ( + injected_position.height.reference.value + == observed_position.height.reference.value + ): + # if the reported height has the same type, compare values + if ( + abs( + injected_position.height.distance + - observed_position.height.distance + ) + > HEIGHT_PRECISION_M + ): + check.record_failed( + "Height reported by Service Provider does not match injected height", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + # If the reported height has a different type, we will check for absurd values below + + if ( + injected_position.height is None + or injected_position.height.reference.value + != observed_position.height.reference.value + ): + # If the injected data does not specify a height or specifies a different type than the observed one, + # we only check for absurd values + if ( + observed_position.height.reference.value + == RIDHeightReference.GroundLevel + ): + if observed_position.height.distance > MAX_HEIGHT_M: + check.record_failed( + "Height above ground reported by Service Provider is unreasonably high and may indicate a bug", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + # Note: for v22a, -1000 is a special value indicating Invalid, No Value or Unknown, so we need to accept it. + if observed_position.height.distance < 0 and ( + observed_position.height.distance != -1000 + or self._rid_version == RIDVersion.f3411_19 + ): + check.record_failed( + "Height above ground reported by Service Provider is negative", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + + elif ( + observed_position.height.reference.value + == RIDHeightReference.TakeoffLocation + ): + if observed_position.height.distance > MAX_HEIGHT_M: + check.record_failed( + "Height above takeoff location reported by Service Provider is unreasonably high and may indicate a bug", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + + # Relative to the takeoff location, negative values are acceptable (ie, flying off and down a mountain) + # and nothing in the standard explicitly forbids them. + if observed_position.height.distance < -MAX_HEIGHT_M: + check.record_failed( + "Height above takeoff location reported by Service Provider is unreasonably low and may indicate a bug", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which is below the minimum of {-MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + else: + raise ValueError( + f"Unexpected height reference value: {observed_position.height.reference.value}" + ) + # Verify that flight details queries succeeded and returned correctly-formatted data for mapping in mappings.values(): details_queries = [ diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 1da64737ba..48a347c29f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -120,6 +120,10 @@ If the timestamp accuracy is not present, the USS under test is not properly imp **[astm.f3411.v19.NET0260,Table1,5](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The observed timestamp accuracy differs from the injected one. +#### ⚠️ Service Provider height check + +**[astm.f3411.v19.NET0260,Table1,13](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The reported height of the flight is unrealistic or otherwise not consistent with the injected data. + #### Successful flight details query check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index fcaeb8160d..4325ab4427 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -120,6 +120,10 @@ If the timestamp accuracy is not present, the USS under test is not properly imp **[astm.f3411.v22a.NET0260,Table1,6](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The observed timestamp accuracy differs from the injected one. +#### ⚠️ Service Provider height check + +**[astm.f3411.v22a.NET0260,Table1,14](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The reported height of the flight is unrealistic or otherwise not consistent with the injected data. + #### Successful flight details query check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 57e3292c61..51d5594221 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -261,6 +261,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,13 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,15 Implemented diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 888e5d045f..ed2e53b28c 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -276,6 +276,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,16 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 38c67b0b46..3d0e662713 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -272,6 +272,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,16 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 5dfcf25353..19b3ea1954 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -273,6 +273,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,16 Implemented