From 68aac3fe92ab03ca0ae4d8f8dc119d2bb074f620 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Tue, 12 Nov 2024 18:02:52 +0100 Subject: [PATCH] [uss_qualifier] netrid: net0260 - check SP correctly reports track (#827) --- .../astm/netrid/display_data_evaluator.py | 32 +++++++++++++++++++ .../astm/netrid/v19/nominal_behavior.md | 4 +++ .../astm/netrid/v22a/nominal_behavior.md | 4 +++ .../suites/astm/netrid/f3411_19.md | 7 +++- .../suites/astm/netrid/f3411_22a.md | 7 +++- .../suites/uspace/network_identification.md | 7 +++- .../suites/uspace/required_services.md | 7 +++- 7 files changed, 64 insertions(+), 4 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 4346ffb3eb..c8e50e2fa1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -976,6 +976,38 @@ def _evaluate_normal_sp_observation( details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with speed accuracy {injected_telemetry.speed_accuracy.value}, but Service Provider reported speed accuracy {mapping.observed_flight.flight.raw.current_state.speed_accuracy.value} at {mapping.observed_flight.query.query.request.initiated_at}", ) + if mapping.observed_flight.flight.raw.current_state is not None: + with self._test_scenario.check( + "Service Provider track", + [mapping.injected_flight.uss_participant_id], + ) as check: + # 361 is a special value, and if we injected it there is no reason we should see a value that could be rounded down to 360 + if math.isclose(injected_telemetry.track, 361): + if ( + abs( + injected_telemetry.track + - mapping.observed_flight.flight.raw.current_state.track + ) + > 0.5 + ): + check.record_failed( + "Special value for track reported by Service Provider does not match injected special value (361)", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with track={injected_telemetry.track}, but Service Provider reported track={mapping.observed_flight.flight.raw.current_state.track} at {mapping.observed_flight.query.query.request.initiated_at}. If this special value is injected we expect it to be reported as is, and do not tolerate the usual track precision of +/- 1 degree.", + ) + + # For any 'normal' track, the resolution is 1 degree. + if ( + abs( + injected_telemetry.track + - mapping.observed_flight.flight.raw.current_state.track + ) + >= 1.0 + ): + check.record_failed( + "Track reported by Service Provider does not match injected track", + details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with track={injected_telemetry.track}, but Service Provider reported track={mapping.observed_flight.flight.raw.current_state.track} at {mapping.observed_flight.query.query.request.initiated_at}", + ) + # Verify that flight details queries succeeded and returned correctly-formatted data for mapping in mappings.values(): details_queries = [ diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 3c30ca44c8..4c19b723ff 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -100,6 +100,10 @@ The identity of flights is determined by precisely matching the known injected p **[astm.f3411.v19.NET0260,Table1,17](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. Injected flight data had a specified speed accuracy that was different from the reported one. +#### ⚠️ Service Provider track check + +**[astm.f3411.v19.NET0260,Table1,18](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. Injected flight data had a specified track that was different from the reported one. + #### Successful flight details query check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index 7c5307f389..60b2b33240 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -100,6 +100,10 @@ The identity of flights is determined by precisely matching the known injected p **[astm.f3411.v22a.NET0260,Table1,18](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. Injected flight data had a specified speed accuracy that was different from the reported one. +#### ⚠️ Service Provider track check + +**[astm.f3411.v22a.NET0260,Table1,19](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider. Injected flight data had a specified track that was different from the reported one. + #### Successful flight details query check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 1a7f49fcd5..048c4f1ecd 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -256,6 +256,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,18 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,19 Implemented diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index fb8e0e17a8..45c0d8c4ec 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -271,6 +271,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,19 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,1a Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index b7ebe4f149..42589096ac 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -17,7 +17,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -267,6 +267,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,19 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,1a Implemented diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index a4139287f7..35bef19799 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -18,7 +18,7 @@ Checked in - astm
.f3411
.v22a + astm
.f3411
.v22a DSS0010 Implemented ASTM NetRID DSS: Token Validation @@ -268,6 +268,11 @@ Implemented ASTM NetRID nominal behavior + + NET0260,Table1,19 + Implemented + ASTM NetRID nominal behavior + NET0260,Table1,1a Implemented