From 32cd7e76be060079a23ef56891472e3bd869df73 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Mon, 4 Nov 2024 13:59:24 +0100 Subject: [PATCH] [uss_qualifier] netrid: dss0130 check observed height --- .../astm/netrid/display_data_evaluator.py | 72 +++++++++++++++++++ .../astm/netrid/v19/nominal_behavior.md | 4 ++ .../astm/netrid/v22a/nominal_behavior.md | 4 ++ .../suites/astm/netrid/f3411_19.md | 7 +- .../suites/astm/netrid/f3411_22a.md | 7 +- .../suites/uspace/network_identification.md | 7 +- .../suites/uspace/required_services.md | 7 +- 7 files changed, 104 insertions(+), 4 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 2970153847..de87b4b6ea 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -6,6 +6,7 @@ import math import s2sphere from s2sphere import LatLng, LatLngRect +from uas_standards.astm.f3411.v22a.api import RIDHeightReference from monitoring.uss_qualifier.scenarios.astm.netrid.common_dictionary_evaluator import ( RIDCommonDictionaryEvaluator, @@ -39,6 +40,10 @@ from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.scenarios.astm.netrid.injection import InjectedFlight +HEIGHT_TOLERANCE_M = 1 +# 20km above ground sounds like a reasonable maximum altitude before declaring it as an error +MAX_HEIGHT_M = 20000 + def _rect_str(rect) -> str: return "({}, {})-({}, {})".format( @@ -918,6 +923,73 @@ def _evaluate_normal_sp_observation( details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with lat={injected_telemetry.position.lat}, lng={injected_telemetry.position.lng}, alt={injected_telemetry.position.alt}, but Service Provider reported lat={observed_position.lat}, lng={observed_position.lng}, alt={observed_position.alt} at {mapping.observed_flight.query.query.request.initiated_at}", ) + # height is an optional field. Evaluate only if present: + if "height" in observed_position: + with self._test_scenario.check("Service Provider height") as check: + if injected_position.height is not None: + # Injected data specifies a height, let's compare: + if ( + injected_position.height.reference.value + == observed_position.height.reference.value + ): + # if the reported height has the same type, compare values + if ( + abs( + injected_position.height.distance + - observed_position.height.distance + ) + > HEIGHT_TOLERANCE_M + ): + check.record_failed( + "Height reported by Service Provider does not match injected height", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + # If the reported height has a different type, we will check for absurd values below + + if injected_position.height is None or injected_position.height.reference.value != observed_position.height.reference.value: + # If the injected data does not specify a height or specifies a different type than the observed one, + # we only check for absurd values + if ( + observed_position.height.reference.value + == RIDHeightReference.GroundLevel + ): + if observed_position.height.distance > MAX_HEIGHT_M: + check.record_failed( + "Height above ground reported by Service Provider is unreasonably high and may indicate a bug", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + # Note: for v22a, -1000 is a special value indicating Invalid, No Value or Unknown + if ( + observed_position.height.distance < 0 + and observed_position.height.distance != -1000 + ): + check.record_failed( + "Height above ground reported by Service Provider is negative", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + + elif ( + observed_position.height.reference.value + == RIDHeightReference.TakeoffLocation + ): + if observed_position.height.distance > MAX_HEIGHT_M: + check.record_failed( + "Height above takeoff location reported by Service Provider is unreasonably high and may indicate a bug", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + + # Relative to the takeoff location, negative values are acceptable (ie, flying off and down a mountain) + # and nothing in the standard explicitly forbids them. + if observed_position.height.distance < -MAX_HEIGHT_M: + check.record_failed( + "Height above takeoff location reported by Service Provider is unreasonably low and may indicate a bug", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which is below the minimum of {-MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + else: + raise ValueError( + f"Unexpected height reference value: {observed_position.height.reference.value}" + ) + # Verify that flight details queries succeeded and returned correctly-formatted data for mapping in mappings.values(): details_queries = [ diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 1795378ab3..f2dd6af98a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -88,6 +88,10 @@ The identity of flights is determined by precisely matching the known injected p **[astm.f3411.v19.NET0260,Table1,11](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. Injected flight data had known altitudes, but the altitude reported by the Service Provider did not match those known altitudes. +#### ⚠️ Service Provider height check + +**[astm.f3411.v19.NET0260,Table1,13](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The reported height of the flight is unrealistic or otherwise not consistent with the injected data. + #### Successful flight details query check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index f7a1c174e7..f7ec1fa0b8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -88,6 +88,10 @@ The identity of flights is determined by precisely matching the known injected p **[astm.f3411.v22a.NET0260,Table1,12](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. Injected flight data had known altitudes, but the altitude reported by the Service Provider did not match those known altitudes. +#### ⚠️ Service Provider height check + +**[astm.f3411.v22a.NET0260,Table1,14](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The reported height of the flight is unrealistic or otherwise not consistent with the injected data. + #### Successful flight details query check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 89bab93543..02843f3a47 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -251,6 +251,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,13 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,9 Implemented diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 3066d6a267..005145d7e9 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -266,6 +266,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,1a Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 6d17e3c0e2..5947fa702c 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -17,7 +17,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -262,6 +262,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,1a Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 4e201b2411..4293a2d84b 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -18,7 +18,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -263,6 +263,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,1a Implemented