diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py index b64aa4d11d..4992420cb1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py @@ -216,8 +216,11 @@ def fail_check(): def evaluate_sp_details(self, details: FlightDetails, participants: List[str]): self._evaluate_uas_id(details.raw.get("uas_id"), participants) - self._evaluate_operator_id(details.operator_id, participants) + self._evaluate_operator_id(None, details.operator_id, participants) self._evaluate_operator_location( + None, + None, + None, details.operator_location, details.operator_altitude, details.operator_altitude_type, @@ -226,6 +229,7 @@ def evaluate_sp_details(self, details: FlightDetails, participants: List[str]): def evaluate_dp_details( self, + injected_details: injection.RIDFlightDetails, observed_details: Optional[observation_api.GetDetailsResponse], participants: List[str], ): @@ -233,19 +237,31 @@ def evaluate_dp_details( return self._evaluate_arbitrary_uas_id( - observed_details.get("uas", {}).get("id"), participants + injected_details.get( + "uas_id", injected_details.get("serial_number", None) + ), # fall back on seria number if no UAS ID + observed_details.get("uas", {}).get("id", None), + participants, ) - operator = observed_details.get("operator", {}) - self._evaluate_operator_id(operator.get("id"), participants) + operator_obs = observed_details.get("operator", {}) + + self._evaluate_operator_id( + injected_details.operator_id, operator_obs.get("id", None), participants + ) - operator_location = operator.get("location", {}) - operator_altitude = operator.get("altitude", {}) - operator_altitude_value = operator_altitude.get("altitude") + operator_altitude_obs = operator_obs.get("altitude", {}) + operator_altitude_value_obs = operator_altitude_obs.get("altitude") + operator_altitude_inj = injected_details.get("operator_altitude", {}) self._evaluate_operator_location( - operator_location, - Altitude.w84m(value=operator_altitude_value), - operator_altitude.get("altitude_type"), + injected_details.get("operator_location", None), + operator_altitude_inj.get( + "altitude", None + ), # should be of the correct type already + operator_altitude_inj.get("altitude_type", None), + operator_obs.get("location", None), + Altitude.w84m(value=operator_altitude_value_obs), + operator_altitude_obs.get("altitude_type", None), participants, ) @@ -296,24 +312,37 @@ def _evaluate_uas_id(self, value: Optional[UASID], participants: List[str]): message=f"Unsupported version {self._rid_version}: skipping UAS ID evaluation", ) - def _evaluate_arbitrary_uas_id(self, value: str, participants: List[str]): + def _evaluate_arbitrary_uas_id( + self, value_inj: str, value_obs: str, participants: List[str] + ): if self._rid_version == RIDVersion.f3411_22a: with self._test_scenario.check( "UAS ID presence in flight details", participants ) as check: - if not value: + if value_obs is None: check.record_failed( - f"UAS ID not present as required by the Common Dictionary definition: {value}", + f"UAS ID not present as required by the Common Dictionary definition: {value_obs}", severity=Severity.Medium, ) return - if SerialNumber(value).valid: + if SerialNumber(value_obs).valid: self._test_scenario.check( "UAS ID (Serial Number format) consistency with Common Dictionary", participants, ).record_passed(participants) + if value_obs is not None: + with self._test_scenario.check( + "UAS ID is consistent with injected one", participants + ) as check: + if value_inj != value_obs: + check.record_failed( + "Observed UAS ID not consistent with injected one", + details=f"Observed: {value_obs} - injected: {value_inj}", + severity=Severity.Medium, + ) + # TODO: Add registration id format check # TODO: Add utm id format check # TODO: Add specific session id format check @@ -360,7 +389,7 @@ def _evaluate_timestamp( if abs(t_inj.datetime - t_obs.datetime).total_seconds() > 1.0: check.record_failed( "Observed timestamp inconsistent with injected one", - details=f"Injected timestamp: {timestamp_inj} – Observed one: {timestamp_obs}", + details=f"Injected timestamp: {timestamp_inj} - Observed one: {timestamp_obs}", severity=Severity.Medium, ) else: @@ -369,18 +398,35 @@ def _evaluate_timestamp( message=f"Unsupported version {self._rid_version}: skipping timestamp evaluation", ) - def _evaluate_operator_id(self, value: Optional[str], participants: List[str]): + def _evaluate_operator_id( + self, + value_inj: Optional[str], + value_obs: Optional[str], + participants: List[str], + ): if self._rid_version == RIDVersion.f3411_22a: - if value: + if value_obs: with self._test_scenario.check( "Operator ID consistency with Common Dictionary", participants ) as check: - is_ascii = all([0 <= ord(c) < 128 for c in value]) + is_ascii = all([0 <= ord(c) < 128 for c in value_obs]) if not is_ascii: check.record_failed( "Operator ID contains non-ascii characters", severity=Severity.Medium, ) + + if value_inj is not None: + with self._test_scenario.check( + "Operator ID is consistent with injected one", participants + ) as check: + if value_inj != value_obs: + check.record_failed( + "Observed Operator ID not consistent with injected one", + details=f"Observed: {value_obs} - injected: {value_inj}", + severity=Severity.Medium, + ) + else: self._test_scenario.record_note( key="skip_reason", @@ -418,7 +464,7 @@ def _evaluate_speed( if abs(speed_obs - speed_inj) > 0.125: check.record_failed( "Observed speed different from injected speed", - details=f"Injected speed was {speed_inj} – observed speed is {speed_obs}", + details=f"Injected speed was {speed_inj} - observed speed is {speed_obs}", severity=Severity.Medium, ) else: @@ -462,7 +508,7 @@ def _evaluate_track( if abs_track_diff > 0.5: check.record_failed( "Observed track direction different from injected one", - details=f"Inject track was {track_inj} – observed one is {track_obs}", + details=f"Inject track was {track_inj} - observed one is {track_obs}", severity=Severity.Medium, ) @@ -510,7 +556,7 @@ def _evaluate_position( ): check.record_failed( "Observed position inconsistent with injected one", - details=f"Injected Position: {position_inj} – Observed Position: {position_obs}", + details=f"Injected Position: {position_inj} - Observed Position: {position_obs}", severity=Severity.Medium, ) else: @@ -551,7 +597,7 @@ def _evaluate_height( ): check.record_failed( "Observed Height is inconsistent with injected one", - details=f"Observed height: {height_obs} – injected: {height_inj}", + details=f"Observed height: {height_obs} - injected: {height_inj}", severity=Severity.Medium, ) else: @@ -562,24 +608,27 @@ def _evaluate_height( def _evaluate_operator_location( self, - position: Optional[LatLngPoint], - altitude: Optional[Altitude], - altitude_type: Optional[observation_api.OperatorAltitudeAltitudeType], + position_inj: Optional[LatLngPoint], + altitude_inj: Optional[Altitude], + altitude_type_inj: Optional[injection.OperatorAltitudeAltitudeType], + position_obs: Optional[LatLngPoint], + altitude_obs: Optional[Altitude], + altitude_type_obs: Optional[observation_api.OperatorAltitudeAltitudeType], participants: List[str], ): if self._rid_version == RIDVersion.f3411_22a: with self._test_scenario.check( "Operator Location consistency with Common Dictionary", participants ) as check: - if not position: + if not position_obs: check.record_failed( "Missing Operator Location position", - details=f"Invalid position: {position}", + details=f"Invalid position: {position_obs}", severity=Severity.Medium, ) return - lat = position.lat + lat = position_obs.lat try: lat = validate_lat(lat) except ValueError: @@ -588,17 +637,33 @@ def _evaluate_operator_location( details=f"Invalid latitude: {lat}", severity=Severity.Medium, ) - lng = position.lng + lng = position_obs.lng try: lng = validate_lng(lng) + position_valid = True except ValueError: + position_valid = False check.record_failed( "Operator Location contains an invalid longitude", details=f"Invalid longitude: {lng}", severity=Severity.Medium, ) - alt = altitude + if position_valid and position_obs is not None and position_inj is not None: + with self._test_scenario.check( + "Operator Location is consistent with injected one", participants + ) as check: + if ( + abs(position_obs.lat - position_inj.lat) > 0.01 + or abs(position_obs.lng - position_obs.lng) > 0.01 + ): + check.record_failed( + summary="Operator Location not consistent with injected one", + details=f"Observed: {position_obs} - injected: {position_inj}", + severity=Severity.Medium, + ) + + alt = altitude_obs if alt: with self._test_scenario.check( "Operator Altitude consistency with Common Dictionary", @@ -616,8 +681,24 @@ def _evaluate_operator_location( details=f"Invalid Operator Altitude units: {alt.units}", severity=Severity.Medium, ) + if altitude_inj is not None: + + with self._test_scenario.check( + "Operator Altitude is consistent with injected one", + participants, + ) as check: + if ( + alt.units != altitude_inj.units + or alt.reference != altitude_inj.reference + or abs(alt.value - altitude_inj.value) > 1 + ): + check.record_failed( + "Observed operator altitude inconsistent with injected one", + details=f"Observed: {alt} - injected: {altitude_inj}", + severity=Severity.Medium, + ) - alt_type = altitude_type + alt_type = altitude_type_obs if alt_type: with self._test_scenario.check( "Operator Altitude Type consistency with Common Dictionary", @@ -634,6 +715,18 @@ def _evaluate_operator_location( severity=Severity.Medium, ) + if altitude_type_inj is not None: + with self._test_scenario.check( + "Operator Altitude Type is consistent with injected one", + participants, + ) as check: + if alt_type != altitude_type_inj: + check.record_failed( + "Observed Operator Altitude Type is inconsistent with injected one", + details=f"Observed: {alt_type} - Injected: {altitude_type_inj}", + severity=Severity.Medium, + ) + else: self._test_scenario.record_note( key="skip_reason", @@ -670,7 +763,7 @@ def _evaluate_operational_status( if not value_obs == value_inj: check.record_failed( "Observed operational status inconsistent with injected one", - details=f"Injected operational status: {value_inj} – Observed {value_obs}", + details=f"Injected operational status: {value_inj} - Observed {value_obs}", ) else: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py index cc06898270..f87c9bdc5e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py @@ -1,54 +1,60 @@ from datetime import datetime, timedelta, timezone -import s2sphere from typing import List, Tuple, Optional +import s2sphere from implicitdict import StringBasedDateTime +from uas_standards.astm.f3411 import v22a +from uas_standards.astm.f3411.v22a.api import ( + Altitude, + LatLngPoint, + UAType, +) +from uas_standards.astm.f3411.v22a.constants import SpecialTrackDirection from uas_standards.interuss.automated_testing.rid.v1 import injection from uas_standards.interuss.automated_testing.rid.v1.observation import ( OperatorAltitudeAltitudeType, RIDHeight, - RIDHeightReference, ) -from uas_standards.astm.f3411.v22a.constants import SpecialTrackDirection -from monitoring.monitorlib.rid import RIDVersion from monitoring.monitorlib.fetch.rid import Flight +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.resources.netrid.evaluation import EvaluationConfiguration from monitoring.uss_qualifier.scenarios.astm.netrid.common_dictionary_evaluator import ( RIDCommonDictionaryEvaluator, ) from monitoring.uss_qualifier.scenarios.interuss.unit_test import UnitTestScenario -from monitoring.uss_qualifier.resources.netrid.evaluation import EvaluationConfiguration -from uas_standards.astm.f3411.v22a.api import ( - Altitude, - LatLngPoint, - UAType, -) -from uas_standards.astm.f3411 import v22a -def _assert_operator_id(value: str, outcome: bool): +def _assert_operator_id(value_inj: str, value_obs: str, outcome: bool): def step_under_test(self: UnitTestScenario): evaluator = RIDCommonDictionaryEvaluator( config=EvaluationConfiguration(), test_scenario=self, rid_version=RIDVersion.f3411_22a, ) - evaluator._evaluate_operator_id(value, []) + evaluator._evaluate_operator_id(value_inj, value_obs, []) unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() assert unit_test_scenario.get_report().successful == outcome def test_operator_id_non_ascii(): - _assert_operator_id("non_ascii©", False) + _assert_operator_id("non_ascii©", "non_ascii©", False) def test_operator_id_ascii(): - _assert_operator_id("ascii.1234", True) + _assert_operator_id("ascii.1234", "ascii.1234", True) def _assert_operator_location( - position, altitude, altitude_type, expected_passed_checks, expected_failed_checks + position_inj, + altitude_inj, + altitude_type_inj, + position, + altitude, + altitude_type, + expected_passed_checks, + expected_failed_checks, ): def step_under_test(self: UnitTestScenario): evaluator = RIDCommonDictionaryEvaluator( @@ -56,7 +62,15 @@ def step_under_test(self: UnitTestScenario): test_scenario=self, rid_version=RIDVersion.f3411_22a, ) - evaluator._evaluate_operator_location(position, altitude, altitude_type, []) + evaluator._evaluate_operator_location( + position_inj, + altitude_inj, + altitude_type_inj, + position, + altitude, + altitude_type, + [], + ) unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() assert ( @@ -72,6 +86,9 @@ def step_under_test(self: UnitTestScenario): def test_operator_location(): valid_locations: List[ Tuple[ + Optional[LatLngPoint], + Optional[Altitude], + Optional[OperatorAltitudeAltitudeType], Optional[LatLngPoint], Optional[Altitude], Optional[OperatorAltitudeAltitudeType], @@ -82,13 +99,19 @@ def test_operator_location(): LatLngPoint(lat=1.0, lng=1.0), None, None, - 1, + LatLngPoint(lat=1.0, lng=1.0), + None, + None, + 2, ), ( LatLngPoint(lat=-90.0, lng=180.0), None, None, - 1, + LatLngPoint(lat=-90.0, lng=180.0), + None, + None, + 2, ), ( LatLngPoint( @@ -97,7 +120,13 @@ def test_operator_location(): ), Altitude(value=1), OperatorAltitudeAltitudeType("Takeoff"), - 3, + LatLngPoint( + lat=46.2, + lng=6.1, + ), + Altitude(value=1), + OperatorAltitudeAltitudeType("Takeoff"), + 6, ), ] for valid_location in valid_locations: @@ -116,10 +145,19 @@ def test_operator_location(): LatLngPoint(lat=-90.001, lng=0), # out of range and valid None, None, - 0, + LatLngPoint(lat=-90.001, lng=0), # out of range and valid + None, + None, + 1, 1, ), ( + LatLngPoint( + lat=0, # valid + lng=180.001, # out of range + ), + None, + None, LatLngPoint( lat=0, # valid lng=180.001, # out of range @@ -130,6 +168,9 @@ def test_operator_location(): 1, ), ( + LatLngPoint(lat=-90.001, lng=180.001), # both out of range + None, + None, LatLngPoint(lat=-90.001, lng=180.001), # both out of range None, None, @@ -137,6 +178,12 @@ def test_operator_location(): 2, ), ( + LatLngPoint( + lat=46.2, + lng=6.1, + ), + None, + None, LatLngPoint( lat="46°12'7.99 N", # Float required lng="6°08'44.48 E", # Float required @@ -153,7 +200,13 @@ def test_operator_location(): ), Altitude(value=1), "invalid", # Invalid value - 2, + LatLngPoint( + lat=46.2, + lng=6.1, + ), + Altitude(value=1), + "invalid", # Invalid value + 5, 1, ), ( @@ -167,7 +220,17 @@ def test_operator_location(): reference="UNKNOWN", # Invalid value ), "Takeoff", - 2, + LatLngPoint( + lat=46.2, + lng=6.1, + ), + Altitude( + value=1000.9, + units="FT", # Invalid value + reference="UNKNOWN", # Invalid value + ), + "Takeoff", + 5, 2, ), ] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 503c408144..c64257741f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -433,6 +433,7 @@ def _evaluate_normal_observation( # Check details of flights (once per flight) for mapping in mapping_by_injection_id.values(): + with self._test_scenario.check( "Successful details observation", [mapping.injected_flight.uss_participant_id], @@ -441,9 +442,10 @@ def _evaluate_normal_observation( if mapping.observed_flight.id in self._retrieved_flight_details: continue - details, query = observer.observe_flight_details( + details_obs, query = observer.observe_flight_details( mapping.observed_flight.id, self._rid_version ) + self._test_scenario.record_query(query) if query.status_code != 200: @@ -454,9 +456,17 @@ def _evaluate_normal_observation( query_timestamps=[query.request.timestamp], ) else: + telemetry_inj = mapping.injected_flight.flight.telemetry[ + mapping.telemetry_index + ] + # Get details that are expected to be valid for the present telemetry: + details_inj = mapping.injected_flight.flight.get_details( + telemetry_inj.timestamp.datetime + ) self._retrieved_flight_details.add(mapping.observed_flight.id) self._common_dictionary_evaluator.evaluate_dp_details( - details, + details_inj, + details_obs, participants=[ observer.participant_id, ], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index 742f5e7df2..444efe485e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -216,6 +216,10 @@ Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../. **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that if the UAS ID is in serial number format, its format is valid. (**[astm.f3411.v22a.NET0470,Table1,1a](../../../../requirements/astm/f3411/v22a.md)**) +#### UAS ID is consistent with injected one check + +If the UAS ID contained in flight details returned by a display provider does not correspond to the injected one, the DP is not providing accurate data and is thus in breach of **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)** + #### Timestamp consistency with Common Dictionary check **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that timestamps are relative to UTC. (**[astm.f3411.v22a.NET0470,Table1,5](../../../../requirements/astm/f3411/v22a.md)**) @@ -232,6 +236,10 @@ If the timestamp reported for an observation does not correspond to the injected **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall (NET0470) provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the Operator ID, if present, is valid. (**[astm.f3411.v22a.NET0470,Table1,9](../../../../requirements/astm/f3411/v22a.md)**) +#### Operator ID is consistent with injected one check + +If the Operator ID contained in flight details returned by a display provider does not correspond to the injected one, the DP is not providing accurate data and is thus in breach of **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)** + #### Current Position consistency with Common Dictionary check **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the Current Position provided is valid. (**[astm.f3411.v22a.NET0470,Table1,10](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0470,Table1,11](../../../../requirements/astm/f3411/v22a.md)**). If the observed Current Position do not contain valid latitude and longitude, this check will fail. @@ -269,14 +277,27 @@ If the speed reported for an observation does not correspond to the injected one **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the Operator Latitude (**[astm.f3411.v22a.NET0470,Table1,23](../../../../requirements/astm/f3411/v22a.md)**) and Longitude (**[astm.f3411.v22a.NET0470,Table1,24](../../../../requirements/astm/f3411/v22a.md)**), if present, are valid. +#### Operator Location is consistent with injected one check + +If the Operator Location contained in flight details returned by a display provider does not correspond to the injected one, the DP is not providing accurate data and is thus in breach of **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)** + #### Operator Altitude consistency with Common Dictionary check **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that, if present, the Operator Altitude is based on WGS-84 height above ellipsoid (HAE) and is provided in meters. (**[astm.f3411.v22a.NET0470,Table1,25](../../../../requirements/astm/f3411/v22a.md)**) +#### Operator Altitude is consistent with injected one check + +If the Operator Altitude contained in flight details returned by a display provider does not correspond to the injected one, the DP is not providing accurate data and is thus in breach of **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)** + #### Operator Altitude Type consistency with Common Dictionary check **[astm.f3411.v22a.NET0470](../../../../requirements/astm/f3411/v22a.md)** requires that Net-RID Display Provider shall provide access to required and optional fields to Remote ID Display Applications according to the Common Dictionary. This check validates that the Operator Altitude Type is valid, if present. (**[astm.f3411.v22a.NET0470,Table1,26](../../../../requirements/astm/f3411/v22a.md)**) +#### Operator Altitude Type is consistent with injected one check + +If the Operator Altitude Type contained in flight details returned by a display provider does not correspond to the injected one, the DP is not providing accurate data and is thus in breach of **[astm.f3411.v22a.NET0450](../../../../requirements/astm/f3411/v22a.md)** + + ## Cleanup The cleanup phase of this test scenario attempts to remove injected data from all SPs.