[uss_qualifier] nominal_behavior – align requirements on observations…

… with the standard (#374) [uss_qualifier] align requirements on observations with the standard (optional altitude and state)
interuss · Nov 29, 2023 · eae8b9f · eae8b9f
1 parent 940352b
commit eae8b9f
Show file tree

Hide file tree

Showing 10 changed files with 33 additions and 87 deletions.
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py
@@ -5,7 +5,6 @@
 import s2sphere
 from arrow import ParserError
 from implicitdict import StringBasedDateTime
-from loguru import logger
 from uas_standards.ansi_cta_2063_a import SerialNumber
 from uas_standards.astm.f3411 import v22a
 from uas_standards.astm.f3411.v22a.api import UASID
@@ -88,34 +87,27 @@ def evaluate_dp_flight(
         observed_flight: observation_api.Flight,
         participants: List[str],
     ):
-        with self._test_scenario.check("Current state present", participants) as check:
-            if not observed_flight.has_field_with_value("current_state"):
-                check.record_failed(
-                    f"Current state for flight {observed_flight.id}",
-                    details=f"The current state must be specified.",
-                    severity=Severity.High,
-                )
-
-        logger.debug(f"Injected flight: {injected_flight}")
-        logger.debug(f"Observed flight: {observed_flight.current_state}")
+        # If the state is present, we do validate its content,
+        # but its presence is optional
+        if injected_flight.has_field_with_value("current_state"):
+            self._evaluate_speed(
+                injected_flight.speed, observed_flight.current_state.speed, participants
+            )
+            self._evaluate_track(
+                injected_flight.track, observed_flight.current_state.track, participants
+            )
+            self._evaluate_timestamp(
+                injected_flight.timestamp,
+                observed_flight.current_state.timestamp,
+                participants,
+            )
 
-        self._evaluate_speed(
-            injected_flight.speed, observed_flight.current_state.speed, participants
-        )
-        self._evaluate_track(
-            injected_flight.track, observed_flight.current_state.track, participants
-        )
-        self._evaluate_timestamp(
-            injected_flight.timestamp,
-            observed_flight.current_state.timestamp,
-            participants,
-        )
+            # TODO check if worth adding correctness check here, it requires some slight (possibly non-trivial)
+            #  changes in evaluate_sp_flights as well
+            self._evaluate_operational_status(
+                observed_flight.current_state.operational_status, participants
+            )
 
-        # TODO check if worth adding correctness check here, it requires some slight (possibly non-trivial)
-        #  changes in evaluate_sp_flights as well
-        self._evaluate_operational_status(
-            observed_flight.current_state.operational_status, participants
-        )
         self._evaluate_position(
             injected_flight.position, observed_flight.most_recent_position, participants
         )

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py
@@ -358,19 +358,9 @@ def _evaluate_normal_observation(
             observed_position = mapping.observed_flight.most_recent_position
             injected_position = injected_telemetry.position
 
-            with self._test_scenario.check(
-                "Altitude is present", [observer.participant_id]
-            ) as check:
-                if "alt" not in observed_position:
-                    check.record_failed(
-                        summary="Displayed flight is missing altitude",
-                        severity=Severity.Medium,
-                        details=f"Displayed data for injected flight {mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} at {injected_telemetry.timestamp} does not have altitude",
-                    )
-
             if "alt" in observed_position:
                 with self._test_scenario.check(
-                    "Correct up-to-date altitude",
+                    "Correct up-to-date altitude if present",
                     [
                         observer.participant_id,
                         mapping.injected_flight.uss_participant_id,

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md
@@ -132,13 +132,9 @@ The timestamps of the injected telemetry usually start in the future.  If a flig
 
 **[astm.f3411.v19.NET0290](../../../../requirements/astm/f3411/v19.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present.
 
-#### Altitude is present check
+#### Correct up-to-date altitude if present check
 
-Failure to report the altitude of a flight is a violation of **[astm.f3411.v19.NET0470](../../../../requirements/astm/f3411/v19.md)**.
-
-#### Correct up-to-date altitude check
-
-If the observed altitude of a flight does not match the altitude of the injected telemetry, the display provider is not providing precise and up-to-date information, and thus does not respect **[astm.f3411.v19.NET0450](../../../../requirements/astm/f3411/v19.md)**.
+If the observed altitude of a flight is reported, but it does not match the altitude of the injected telemetry, the display provider is not providing precise and up-to-date information, and thus does not respect **[astm.f3411.v19.NET0450](../../../../requirements/astm/f3411/v19.md)**.
 
 #### Area too large check
 
@@ -174,10 +170,6 @@ This check validates that the display area of a cluster, measured and provided i
 
 Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call for flight details is expected to succeed since a valid ID was provided by uss_qualifier.
 
-#### Current state present check
-
-**[astm.f3411.v19.NET0470](../../../../requirements/astm/f3411/v19.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the current state is present. If it is not, this check will fail.
-
 ## Cleanup
 
 The cleanup phase of this test scenario attempts to remove injected data from all SPs.

diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md
@@ -164,13 +164,9 @@ NET0260 requires a SP to provide flights up to *NetMaxNearRealTimeDataPeriod* in
 
 **[astm.f3411.v22a.NET0290](../../../../requirements/astm/f3411/v22a.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present.
 
-#### Altitude is present check
+#### Correct up-to-date altitude if present check
 
-Per **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)**, failure to report the altitude of a flight is a violation of **[astm.f3411.v22a.NET0470,Table1,12](../../../../requirements/astm/f3411/v22a.md)**.
-
-#### Correct up-to-date altitude check
-
-If the observed altitude of a flight does not match the altitude of the injected telemetry, the display provider is not providing precise and up-to-date information, and thus does not respect **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)**.
+If the observed altitude of a flight is reported, but it does not match the altitude of the injected telemetry, the display provider is not providing precise and up-to-date information, and thus does not respect **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)**.
 
 #### Area too large check
 
@@ -206,10 +202,6 @@ This check validates that the display area of a cluster, measured and provided i
 
 Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call for flight details is expected to succeed since a valid ID was provided by uss_qualifier.
 
-#### Current state present check
-
-**[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the current state is present. If it is not, this check will fail.
-
 #### UAS ID presence in flight details check
 
 **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the UAS ID is present in the information sent by the Display Provider. (**[astm.f3411.v22a.NET0470,Table1,1](../../../../requirements/astm/f3411/v22a.md)**)

diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md
@@ -21,7 +21,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="62" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v19.md">astm<br>.f3411<br>.v19</a></td>
+    <td rowspan="61" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v19.md">astm<br>.f3411<br>.v19</a></td>
     <td><a href="../../../requirements/astm/f3411/v19.md">DSS0030,a</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v19/dss_interoperability.md">ASTM F3411-19 NetRID DSS interoperability</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md">ASTM NetRID DSS: ISA Subscription Interactions</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v19/dss/isa_validation.md">ASTM NetRID DSS: Submitted ISA Validations</a><br><a href="../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
@@ -286,11 +286,6 @@
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
-  <tr>
-    <td><a href="../../../requirements/astm/f3411/v19.md">NET0470</a></td>
-    <td>Implemented</td>
-    <td><a href="../../../scenarios/astm/netrid/v19/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
-  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v19.md">NET0480</a></td>
     <td>Implemented</td>

diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md
@@ -21,7 +21,7 @@
     <th><a href="../../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="87" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
+    <td rowspan="86" style="vertical-align:top;"><a href="../../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
     <td><a href="../../../requirements/astm/f3411/v22a.md">DSS0030</a></td>
     <td>Implemented</td>
     <td><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a></td>
@@ -356,11 +356,6 @@
     <td>TODO</td>
     <td><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
-  <tr>
-    <td><a href="../../../requirements/astm/f3411/v22a.md">NET0470,Table1,12</a></td>
-    <td>Implemented</td>
-    <td><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
-  </tr>
   <tr>
     <td><a href="../../../requirements/astm/f3411/v22a.md">NET0470,Table1,15</a></td>
     <td>Implemented</td>
@@ -497,6 +492,6 @@
   <tr>
     <td><a href="../../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
     <td>Implemented</td>
-    <td><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+    <td><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
 </table>
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md
@@ -228,6 +228,6 @@
   <tr>
     <td><a href="../../../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
     <td>Implemented</td>
-    <td><a href="../../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
+    <td><a href="../../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
   </tr>
 </table>
diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md
@@ -409,6 +409,6 @@
   <tr>
     <td><a href="../../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
     <td>Implemented</td>
-    <td><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
+    <td><a href="../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
   </tr>
 </table>
diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md
@@ -16,7 +16,7 @@
     <th><a href="../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="87" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
+    <td rowspan="86" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
     <td><a href="../../requirements/astm/f3411/v22a.md">DSS0030</a></td>
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a></td>
@@ -351,11 +351,6 @@
     <td>TODO</td>
     <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
-  <tr>
-    <td><a href="../../requirements/astm/f3411/v22a.md">NET0470,Table1,12</a></td>
-    <td>Implemented</td>
-    <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
-  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3411/v22a.md">NET0470,Table1,15</a></td>
     <td>Implemented</td>
@@ -492,6 +487,6 @@
   <tr>
     <td><a href="../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
     <td>Implemented</td>
-    <td><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+    <td><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
 </table>
diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md
@@ -18,7 +18,7 @@
     <th><a href="../README.md#checked-in">Checked in</a></th>
   </tr>
   <tr>
-    <td rowspan="87" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
+    <td rowspan="86" style="vertical-align:top;"><a href="../../requirements/astm/f3411/v22a.md">astm<br>.f3411<br>.v22a</a></td>
     <td><a href="../../requirements/astm/f3411/v22a.md">DSS0030</a></td>
     <td>Implemented</td>
     <td><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a></td>
@@ -353,11 +353,6 @@
     <td>TODO</td>
     <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
-  <tr>
-    <td><a href="../../requirements/astm/f3411/v22a.md">NET0470,Table1,12</a></td>
-    <td>Implemented</td>
-    <td><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
-  </tr>
   <tr>
     <td><a href="../../requirements/astm/f3411/v22a.md">NET0470,Table1,15</a></td>
     <td>Implemented</td>
@@ -631,7 +626,7 @@
   <tr>
     <td><a href="../../requirements/interuss/f3411/dss_endpoints.md">SearchISAs</a></td>
     <td>Implemented</td>
-    <td><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../scenarios/astm/netrid/v19/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
+    <td><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a><br><a href="../../scenarios/astm/netrid/v22a/nominal_behavior.md">ASTM NetRID nominal behavior</a></td>
   </tr>
   <tr>
     <td rowspan="1" style="vertical-align:top;"><a href="../../requirements/interuss/f3548/notification_requirements.md">interuss<br>.f3548<br>.notification_requirements</a></td>