From 668b9535c329ff1ff78400be2ff241c62e20d8f3 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Thu, 21 Nov 2024 08:47:03 +0100 Subject: [PATCH 1/2] [uss_qualifier] netrid: net0260 check observed height (#840) --- .../astm/netrid/display_data_evaluator.py | 39 +++++++++++++++++++ .../astm/netrid/v19/nominal_behavior.md | 4 ++ .../astm/netrid/v22a/nominal_behavior.md | 4 ++ .../suites/astm/netrid/f3411_19.md | 7 +++- .../suites/astm/netrid/f3411_22a.md | 7 +++- .../suites/uspace/network_identification.md | 7 +++- .../suites/uspace/required_services.md | 7 +++- 7 files changed, 71 insertions(+), 4 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index fc9312bf3a..43f53b7e76 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -6,6 +6,8 @@ import s2sphere from loguru import logger from s2sphere import LatLng, LatLngRect +from uas_standards.astm.f3411.v22a.api import RIDHeightReference + from uas_standards.interuss.automated_testing.rid.v1.observation import ( Flight, GetDisplayDataResponse, @@ -50,6 +52,7 @@ def _rect_str(rect) -> str: VERTICAL_SPEED_PRECISION = 0.1 SPEED_PRECISION = 0.05 TIMESTAMP_ACCURACY_PRECISION = 0.05 +HEIGHT_PRECISION_M = 1 @dataclass @@ -1088,6 +1091,42 @@ def _evaluate_normal_sp_observation( details=f"Timestamp accuracy in Service Provider {mapping.injected_flight.uss_participant_id}'s response for flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} with telemetry index {mapping.telemetry_index} is {mapping.observed_flight.timestamp_accuracy} which is not equal to the injected value of {injected_telemetry.timestamp_accuracy}", ) + if "height" in injected_position: + # We injected a height so expect to observe one + if "height" not in observed_position: + check.record_failed( + "A value was injected for the height field, but none was returned in Service Provider response", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had a height injected, but Service Provider did not return a height at {mapping.observed_flight.query.query.request.initiated_at}", + ) + else: + if ( + injected_position.height.reference.value + != observed_position.height.reference.value + ): + check.record_failed( + "Height reference reported by Service Provider does not match injected height reference", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + if not math.isclose( + injected_position.height.distance, + observed_position.height.distance, + abs_tol=HEIGHT_PRECISION_M, + ): + check.record_failed( + "Height reported by Service Provider does not match injected height", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + else: + # We did not inject a height, but a height returning the magic 'unknown' value would still be seen as valid + if "height" in observed_position: + if not math.isclose( + observed_position.height.distance, -1000, abs_tol=1 + ): + check.record_failed( + "Injected no height, but Service Provider reported a height", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had no height injected, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + # Verify that flight details queries succeeded and returned correctly-formatted data for mapping in mappings.values(): details_queries = [ diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 1da64737ba..48a347c29f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -120,6 +120,10 @@ If the timestamp accuracy is not present, the USS under test is not properly imp **[astm.f3411.v19.NET0260,Table1,5](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The observed timestamp accuracy differs from the injected one. +#### ⚠️ Service Provider height check + +**[astm.f3411.v19.NET0260,Table1,13](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The reported height of the flight is unrealistic or otherwise not consistent with the injected data. + #### Successful flight details query check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index fcaeb8160d..4325ab4427 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -120,6 +120,10 @@ If the timestamp accuracy is not present, the USS under test is not properly imp **[astm.f3411.v22a.NET0260,Table1,6](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The observed timestamp accuracy differs from the injected one. +#### ⚠️ Service Provider height check + +**[astm.f3411.v22a.NET0260,Table1,14](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. The reported height of the flight is unrealistic or otherwise not consistent with the injected data. + #### Successful flight details query check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 57e3292c61..23b63987b6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -261,6 +261,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,13 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,15 Implemented diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 888e5d045f..e803f28ab1 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -276,6 +276,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,16 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 38c67b0b46..84373e6627 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -17,7 +17,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -272,6 +272,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,16 Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 5dfcf25353..20054687de 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -18,7 +18,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -273,6 +273,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,14 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,16 Implemented From 83211893c9d633a721a4d3b056f529cbf872d402 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Thu, 21 Nov 2024 09:32:02 +0100 Subject: [PATCH 2/2] [interfaces] update automated_testing submodule to latest version (#851) --- interfaces/automated_testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interfaces/automated_testing b/interfaces/automated_testing index 4e07f46eb4..417797f42e 160000 --- a/interfaces/automated_testing +++ b/interfaces/automated_testing @@ -1 +1 @@ -Subproject commit 4e07f46eb4452da761fb1658d3775d801d19312b +Subproject commit 417797f42e8ebd7cc1b71e82ec0fee0fe1e972a1