[uss_qualifier] netrid: dss0130 check observed height

interuss · Nov 18, 2024 · ba5d25e · ba5d25e
1 parent eefa380
commit ba5d25e
Show file tree

Hide file tree

Showing 7 changed files with 104 additions and 0 deletions.
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py
@@ -6,6 +6,8 @@
 import s2sphere
 from loguru import logger
 from s2sphere import LatLng, LatLngRect
+from uas_standards.astm.f3411.v22a.api import RIDHeightReference
+
 from uas_standards.interuss.automated_testing.rid.v1.observation import (
     Flight,
     GetDisplayDataResponse,
@@ -50,6 +52,9 @@ def _rect_str(rect) -> str:
 VERTICAL_SPEED_PRECISION = 0.1
 SPEED_PRECISION = 0.05
 TIMESTAMP_ACCURACY_PRECISION = 0.05
+HEIGHT_PRECISION_M = 1
+# 20km above ground sounds like a reasonable maximum altitude before declaring it as an error
+MAX_HEIGHT_M = 20000
 
 
 @dataclass
@@ -1088,6 +1093,77 @@ def _evaluate_normal_sp_observation(
                             details=f"Timestamp accuracy in Service Provider {mapping.injected_flight.uss_participant_id}'s response for flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} with telemetry index {mapping.telemetry_index} is {mapping.observed_flight.timestamp_accuracy} which is not equal to the injected value of {injected_telemetry.timestamp_accuracy}",
                         )
 
+            # height is an optional field. Evaluate only if present:
+            if "height" in observed_position:
+                with self._test_scenario.check("Service Provider height") as check:
+                    if injected_position.height is not None:
+                        # Injected data specifies a height, let's compare:
+                        if (
+                            injected_position.height.reference.value
+                            == observed_position.height.reference.value
+                        ):
+                            # if the reported height has the same type, compare values
+                            if (
+                                abs(
+                                    injected_position.height.distance
+                                    - observed_position.height.distance
+                                )
+                                > HEIGHT_PRECISION_M
+                            ):
+                                check.record_failed(
+                                    "Height reported by Service Provider does not match injected height",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+                        # If the reported height has a different type, we will check for absurd values below
+
+                    if (
+                        injected_position.height is None
+                        or injected_position.height.reference.value
+                        != observed_position.height.reference.value
+                    ):
+                        # If the injected data does not specify a height or specifies a different type than the observed one,
+                        # we only check for absurd values
+                        if (
+                            observed_position.height.reference.value
+                            == RIDHeightReference.GroundLevel
+                        ):
+                            if observed_position.height.distance > MAX_HEIGHT_M:
+                                check.record_failed(
+                                    "Height above ground reported by Service Provider is unreasonably high and may indicate a bug",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+                            # Note: for v22a, -1000 is a special value indicating Invalid, No Value or Unknown, so we need to accept it.
+                            if observed_position.height.distance < 0 and (
+                                observed_position.height.distance != -1000
+                                or self._rid_version == RIDVersion.f3411_19
+                            ):
+                                check.record_failed(
+                                    "Height above ground reported by Service Provider is negative",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+
+                        elif (
+                            observed_position.height.reference.value
+                            == RIDHeightReference.TakeoffLocation
+                        ):
+                            if observed_position.height.distance > MAX_HEIGHT_M:
+                                check.record_failed(
+                                    "Height above takeoff location reported by Service Provider is unreasonably high and may indicate a bug",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+
+                            # Relative to the takeoff location, negative values are acceptable (ie, flying off and down a mountain)
+                            # and nothing in the standard explicitly forbids them.
+                            if observed_position.height.distance < -MAX_HEIGHT_M:
+                                check.record_failed(
+                                    "Height above takeoff location reported by Service Provider is unreasonably low and may indicate a bug",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which is below the minimum of {-MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+                        else:
+                            raise ValueError(
+                                f"Unexpected height reference value: {observed_position.height.reference.value}"
+                            )
+
         # Verify that flight details queries succeeded and returned correctly-formatted data
         for mapping in mappings.values():
             details_queries = [

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md
@@ -120,6 +120,10 @@ If the timestamp accuracy is not present, the USS under test is not properly imp
 
 **[astm.f3411.v19.NET0260,Table1,5](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  The observed timestamp accuracy differs from the injected one.
 
+#### ⚠️ Service Provider height check
+
+**[astm.f3411.v19.NET0260,Table1,13](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  The reported height of the flight is unrealistic or otherwise not consistent with the injected data.
+
 #### Successful flight details query check
 
 **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint.  This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md
@@ -120,6 +120,10 @@ If the timestamp accuracy is not present, the USS under test is not properly imp
 
 **[astm.f3411.v22a.NET0260,Table1,6](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  The observed timestamp accuracy differs from the injected one.
 
+#### ⚠️ Service Provider height check
+
+**[astm.f3411.v22a.NET0260,Table1,14](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  The reported height of the flight is unrealistic or otherwise not consistent with the injected data.
+
 #### Successful flight details query check
 
 **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint.  This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.

diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md
@@ -261,6 +261,11 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3411/v19.md">NET0260,Table1,13</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v19.md">NET0260,Table1,15</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md
@@ -276,6 +276,11 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3411/v22a.md">NET0260,Table1,14</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v22a.md">NET0260,Table1,16</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md
@@ -272,6 +272,11 @@
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,14</a></td>
+    <td>Implemented</td>
+    <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,16</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md
@@ -273,6 +273,11 @@
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,14</a></td>
+    <td>Implemented</td>
+    <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,16</a></td>
     <td>Implemented</td>