[uss_qualifier] netrid: dss0130 check observed height

interuss · Nov 13, 2024 · 48b160f · 48b160f
1 parent 58e40fe
commit 48b160f
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 4 deletions.
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py
@@ -6,6 +6,8 @@
 import s2sphere
 from loguru import logger
 from s2sphere import LatLng, LatLngRect
+from uas_standards.astm.f3411.v22a.api import RIDHeightReference
+
 from uas_standards.interuss.automated_testing.rid.v1.observation import (
     Flight,
     GetDisplayDataResponse,
@@ -49,6 +51,9 @@ def _rect_str(rect) -> str:
 
 VERTICAL_SPEED_PRECISION = 0.1
 SPEED_PRECISION = 0.05
+HEIGHT_TOLERANCE_M = 1
+# 20km above ground sounds like a reasonable maximum altitude before declaring it as an error
+MAX_HEIGHT_M = 20000
 
 
 @dataclass
@@ -1021,6 +1026,77 @@ def _evaluate_normal_sp_observation(
                             details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with track={injected_telemetry.track}, but Service Provider reported track={mapping.observed_flight.flight.raw.current_state.track} at {mapping.observed_flight.query.query.request.initiated_at}",
                         )
 
+            # height is an optional field. Evaluate only if present:
+            if "height" in observed_position:
+                with self._test_scenario.check("Service Provider height") as check:
+                    if injected_position.height is not None:
+                        # Injected data specifies a height, let's compare:
+                        if (
+                            injected_position.height.reference.value
+                            == observed_position.height.reference.value
+                        ):
+                            # if the reported height has the same type, compare values
+                            if (
+                                abs(
+                                    injected_position.height.distance
+                                    - observed_position.height.distance
+                                )
+                                > HEIGHT_TOLERANCE_M
+                            ):
+                                check.record_failed(
+                                    "Height reported by Service Provider does not match injected height",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+                        # If the reported height has a different type, we will check for absurd values below
+
+                    if (
+                        injected_position.height is None
+                        or injected_position.height.reference.value
+                        != observed_position.height.reference.value
+                    ):
+                        # If the injected data does not specify a height or specifies a different type than the observed one,
+                        # we only check for absurd values
+                        if (
+                            observed_position.height.reference.value
+                            == RIDHeightReference.GroundLevel
+                        ):
+                            if observed_position.height.distance > MAX_HEIGHT_M:
+                                check.record_failed(
+                                    "Height above ground reported by Service Provider is unreasonably high and may indicate a bug",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+                            # Note: for v22a, -1000 is a special value indicating Invalid, No Value or Unknown, so we need to accept it.
+                            if observed_position.height.distance < 0 and (
+                                observed_position.height.distance != -1000
+                                or self._rid_version == RIDVersion.f3411_19
+                            ):
+                                check.record_failed(
+                                    "Height above ground reported by Service Provider is negative",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, but Service Provider reported height={observed_position.height.distance} {observed_position.height.reference.value} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+
+                        elif (
+                            observed_position.height.reference.value
+                            == RIDHeightReference.TakeoffLocation
+                        ):
+                            if observed_position.height.distance > MAX_HEIGHT_M:
+                                check.record_failed(
+                                    "Height above takeoff location reported by Service Provider is unreasonably high and may indicate a bug",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which exceeds the maximum of {MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+
+                            # Relative to the takeoff location, negative values are acceptable (ie, flying off and down a mountain)
+                            # and nothing in the standard explicitly forbids them.
+                            if observed_position.height.distance < -MAX_HEIGHT_M:
+                                check.record_failed(
+                                    "Height above takeoff location reported by Service Provider is unreasonably low and may indicate a bug",
+                                    details=f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} with height={injected_position.height.distance} {injected_position.height.reference.value}, which is below the minimum of {-MAX_HEIGHT_M} at {mapping.observed_flight.query.query.request.initiated_at}",
+                                )
+                        else:
+                            raise ValueError(
+                                f"Unexpected height reference value: {observed_position.height.reference.value}"
+                            )
+
         # Verify that flight details queries succeeded and returned correctly-formatted data
         for mapping in mappings.values():
             details_queries = [

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md
@@ -108,6 +108,10 @@ The identity of flights is determined by precisely matching the known injected p
 
 **[astm.f3411.v19.NET0260,Table1,15](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  Injected flight data had a specified geodetic altitude accuracy that was different from the reported one.
 
+#### ⚠️ Service Provider height check
+
+**[astm.f3411.v19.NET0260,Table1,13](../../../../requirements/astm/f3411/v19.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  The reported height of the flight is unrealistic or otherwise not consistent with the injected data.
+
 #### Successful flight details query check
 
 **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint.  This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md
@@ -108,6 +108,10 @@ The identity of flights is determined by precisely matching the known injected p
 
 **[astm.f3411.v22a.NET0260,Table1,16](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  Injected flight data had a specified geodetic altitude accuracy that was different from the reported one.
 
+#### ⚠️ Service Provider height check
+
+**[astm.f3411.v22a.NET0260,Table1,14](../../../../requirements/astm/f3411/v22a.md)** requires that relevant Remote ID data, consistent with the common data dictionary, be reported by the Service Provider.  The reported height of the flight is unrealistic or otherwise not consistent with the injected data.
+
 #### Successful flight details query check
 
 **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint.  This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully.

diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md
@@ -21,7 +21,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="67" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v19.md">astm<br>.f3411<br>.v19</a></td>
+    <td rowspan="68" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v19.md">astm<br>.f3411<br>.v19</a></td>
     <td><a href="../../../requirements/astm/f3411/v19.md">DSS0010</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
@@ -251,6 +251,11 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3411/v19.md">NET0260,Table1,13</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v19.md">NET0260,Table1,15</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md
@@ -21,7 +21,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="92" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
+    <td rowspan="93" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
     <td><a href="../../../requirements/astm/f3411/v22a.md">DSS0010</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
@@ -266,6 +266,11 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../../requirements/astm/f3411/v22a.md">NET0260,Table1,14</a></td>
+    <td>Implemented</td>
+    <td><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v22a.md">NET0260,Table1,16</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md
@@ -17,7 +17,7 @@
     <th><a href="../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="92" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
+    <td rowspan="93" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
     <td><a href="../../requirements/astm/f3411/v22a.md">DSS0010</a></td>
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
@@ -262,6 +262,11 @@
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,14</a></td>
+    <td>Implemented</td>
+    <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,16</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md
@@ -18,7 +18,7 @@
     <th><a href="../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="92" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
+    <td rowspan="93" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
     <td><a href="../../requirements/astm/f3411/v22a.md">DSS0010</a></td>
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
@@ -263,6 +263,11 @@
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
+  <tr>
+    <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,14</a></td>
+    <td>Implemented</td>
+    <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3411/v22a.md">NET0260,Table1,16</a></td>
     <td>Implemented</td>