From 13f62fbe0cba095771ccd41ab3c33fa91ce6745c Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Wed, 3 Jan 2024 18:58:49 +0000 Subject: [PATCH] [uss_qualifier] Implement MSL altitude scenario (#450) Implement MSL altitude scenario --- .../mock_uss/riddp/routes_observation.py | 14 ++- monitoring/monitorlib/fetch/__init__.py | 8 ++ .../resources/netrid/observers.py | 2 + .../scenarios/uspace/netrid/msl.md | 6 +- .../scenarios/uspace/netrid/msl.py | 118 +++++++++++++++++- monitoring/uss_qualifier/suites/suite.py | 27 ++++ requirements.txt | 2 +- .../monitoring/monitorlib/fetch/Query.json | 4 +- 8 files changed, 171 insertions(+), 10 deletions(-) diff --git a/monitoring/mock_uss/riddp/routes_observation.py b/monitoring/mock_uss/riddp/routes_observation.py index 5f3a90942f..6c6f46cc69 100644 --- a/monitoring/mock_uss/riddp/routes_observation.py +++ b/monitoring/mock_uss/riddp/routes_observation.py @@ -19,6 +19,10 @@ ) from monitoring.mock_uss import webapp from monitoring.mock_uss.auth import requires_scope +from uas_standards.interuss.automated_testing.rid.v1.observation import ( + AltitudeReference, + MSLAltitude, +) from . import clustering, database, utm_client from .behavior import DisplayProviderBehavior from .config import KEY_RID_VERSION @@ -62,8 +66,8 @@ def _make_flight_observation( paths.append(current_path) p = flight.most_recent_position - msl_alt = p.alt - egm96_geoid_offset(s2sphere.LatLng.from_degrees(p.lat, p.lng)) - # TODO: Return msl_alt in observation information + msl_alt_m = p.alt - egm96_geoid_offset(s2sphere.LatLng.from_degrees(p.lat, p.lng)) + msl_alt = MSLAltitude(meters=msl_alt_m, reference_datum=AltitudeReference.EGM96) current_state = observation_api.CurrentState( timestamp=p.time.isoformat(), operational_status=flight.operational_status, @@ -76,7 +80,11 @@ def _make_flight_observation( return observation_api.Flight( id=flight.id, most_recent_position=observation_api.Position( - lat=p.lat, lng=p.lng, alt=p.alt, height=h + lat=p.lat, + lng=p.lng, + alt=p.alt, + height=h, + msl_alt=msl_alt, ), recent_paths=[observation_api.Path(positions=path) for path in paths], current_state=current_state, diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 5f08013028..279e9c43bd 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -254,6 +254,14 @@ class QueryType(str, Enum): "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" ) + # InterUSS RID observation interface + InterUSSRIDObservationV1GetDisplayData = ( + "interuss.automated_testing.rid.v1.observation.getDisplayData" + ) + InterUSSRIDObservationV1GetDetails = ( + "interuss.automated_testing.rid.v1.observation.getDetails" + ) + def __str__(self): return self.value diff --git a/monitoring/uss_qualifier/resources/netrid/observers.py b/monitoring/uss_qualifier/resources/netrid/observers.py index fa2c0b7b89..d202fd05d9 100644 --- a/monitoring/uss_qualifier/resources/netrid/observers.py +++ b/monitoring/uss_qualifier/resources/netrid/observers.py @@ -49,6 +49,7 @@ def observe_system( url, scope=Scope.Observe, participant_id=self.participant_id, + query_type=QueryType.InterUSSRIDObservationV1GetDisplayData, ) try: result = ( @@ -72,6 +73,7 @@ def observe_flight_details( f"/display_data/{flight_id}", scope=Scope.Observe, participant_id=self.participant_id, + query_type=QueryType.InterUSSRIDObservationV1GetDetails, ) # Record query metadata for later use in the aggregate checks query.participant_id = self.participant_id diff --git a/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md index 1307c69742..b869850d08 100644 --- a/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md +++ b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.md @@ -41,7 +41,11 @@ If an appropriate test report cannot be found, this scenario will be discontinue #### ⚠️ Message contains MSL altitude check -If the response message for the remote identification observation made by the virtual/automated authorised user does not contain the UAS's MSL altitude, the USSP will have failed to comply with **[uspace.article8.MSLAltitude](../../../requirements/uspace/article8.md)**. +If a response message for the remote identification observation made by the virtual/automated authorised user does not contain the UAS's MSL altitude, the USSP will have failed to comply with **[uspace.article8.MSLAltitude](../../../requirements/uspace/article8.md)**. + +#### ⚠️ MSL altitude is reported using an acceptable datum check + +While the WGS84 ellipsoid is one approximation of sea level, it is not included as a recommended definition of mean sea level in the Guidance Material. If the USSP reports MSL altitude relative to a datum not recommended in the Guidance Material, the USSP will have failed to comply with **[uspace.article8.MSLAltitude](../../../requirements/uspace/article8.md)** according to the Guidance Material. #### ⚠️ MSL altitude is correct check diff --git a/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py index c90277688c..75cef29b7c 100644 --- a/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py +++ b/monitoring/uss_qualifier/scenarios/uspace/netrid/msl.py @@ -1,10 +1,29 @@ -from typing import List +from dataclasses import dataclass +from datetime import datetime +from typing import List, Dict, Set +from implicitdict import ImplicitDict +import s2sphere + +from uas_standards.interuss.automated_testing.rid.v1.observation import ( + AltitudeReference, + GetDisplayDataResponse, +) + +from monitoring.monitorlib.fetch import Query, QueryType +from monitoring.monitorlib.geo import egm96_geoid_offset from monitoring.uss_qualifier.configurations.configuration import ParticipantID from monitoring.uss_qualifier.resources.netrid import NetRIDObserversResource +from monitoring.uss_qualifier.scenarios.astm.netrid.common.nominal_behavior import ( + NominalBehavior, +) from monitoring.uss_qualifier.scenarios.scenario import TestScenario +MAXIMUM_MSL_ERROR_M = 0.5 # meters maximum difference between expected MSL altitude and reported MSL altitude +ACCEPTABLE_DATUMS = {AltitudeReference.EGM96, AltitudeReference.EGM2008} + + class MSLAltitude(TestScenario): _ussps: List[ParticipantID] @@ -18,14 +37,105 @@ def run(self, context): self.begin_test_case("UAS observations evaluation") self.begin_test_step("Find nominal behavior report") - # TODO: Find test report for NetRID nominal behavior scenario + reports = context.find_test_scenario_reports(NominalBehavior) self.end_test_step() + if not reports: + self.record_note( + "Skip reason", + f"Nominal behavior test scenario report could not be found for any of the scenario types {', '.join(SCENARIO_TYPES)}", + ) + self.end_test_scenario() + return + self.begin_test_step("Evaluate UAS observations") - # TODO: Examine observation queries in test report to see if MSL was present - # TODO: When MSL is present, verify that its value matches injected altitude above ellipsoid + for report in reports: + self._evaluate_msl_altitude(report.queries()) self.end_test_step() self.end_test_case() self.end_test_scenario() + + def _evaluate_msl_altitude(self, queries: List[Query]): + for query in queries: + if ( + "query_type" not in query + or query.query_type != QueryType.InterUSSRIDObservationV1GetDisplayData + ): + continue + if "json" not in query.response or not query.response.json: + # Invalid observation; this should already have been recorded as a failure + continue + try: + resp: GetDisplayDataResponse = ImplicitDict.parse( + query.response.json, GetDisplayDataResponse + ) + except ValueError as e: + # Invalid observation; this should already have been recorded as a failure + continue + if "flights" not in resp or not resp.flights: + continue + + self.record_query(query) + participant_id = query.participant_id if "participant_id" in query else None + q = query.request.timestamp + for flight in resp.flights: + with self.check( + "Message contains MSL altitude", participant_id + ) as check: + if ( + "msl_alt" not in flight.most_recent_position + or flight.most_recent_position.msl_alt is None + ): + check.record_failed( + summary="MSL altitude missing from observation", + details=f"Flight {flight.id} was missing `msl_alt` field in the noted RID observation", + query_timestamps=[q], + ) + continue + + with self.check( + "MSL altitude is reported using an acceptable datum", participant_id + ) as check: + if ( + "reference_datum" not in flight.most_recent_position.msl_alt + or flight.most_recent_position.msl_alt.reference_datum + not in ACCEPTABLE_DATUMS + ): + check.record_failed( + summary=f"MSL altitude reported relative to {flight.most_recent_position.msl_alt.reference_datum.value} rather than {' or '.join(ACCEPTABLE_DATUMS)}", + details=f"The only acceptable MSL altitude datums for U-space are {' or '.join(ACCEPTABLE_DATUMS)}, and {flight.most_recent_position.msl_alt.reference_datum.value} reported for flight {flight.id} is not one of them", + query_timestamps=[q], + ) + continue + + if ( + "alt" in flight.most_recent_position + and flight.most_recent_position is not None + ): + with self.check("MSL altitude is correct", participant_id) as check: + geoid_offset = egm96_geoid_offset( + s2sphere.LatLng.from_degrees( + flight.most_recent_position.lat, + flight.most_recent_position.lng, + ) + ) + expected_msl_alt = ( + flight.most_recent_position.alt - geoid_offset + ) + if ( + abs( + expected_msl_alt + - flight.most_recent_position.msl_alt.meters + ) + > MAXIMUM_MSL_ERROR_M + ): + check.record_failed( + summary=f"Reported MSL altitude {flight.most_recent_position.msl_alt.meters:.1f}m does not match expected MSL altitude {expected_msl_alt:.1f}m", + details=f"Altitude for flight {flight.id} at {flight.most_recent_position.lat}, {flight.most_recent_position.lng} was reported as {flight.most_recent_position.alt} meters above the WGS84 ellipsoid, and the EGM96 geoid is {geoid_offset} meters above the WGS84 ellipsoid at this point, but the MSL altitude was reported as {flight.most_recent_position.msl_alt.meters} meters above {flight.most_recent_position.msl_alt.reference_datum} rather than the expected {expected_msl_alt} meters", + query_timestamps=[q], + ) + else: + pass + # TODO: check MSL altitude against injection if WGS84 altitude is not specified in observation diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index a41e0b2ebe..10608f8137 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -6,6 +6,7 @@ import json import re from typing import Dict, List, Optional, Union, Iterator +from typing import Type import arrow @@ -52,6 +53,7 @@ ScenarioCannotContinueError, TestRunCannotContinueError, ) +from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name from monitoring.uss_qualifier.suites.definitions import ( TestSuiteActionDeclaration, TestSuiteDefinition, @@ -407,6 +409,31 @@ def sibling_queries(self) -> Iterator[Query]: for q in child.report.queries(): yield q + def find_test_scenario_reports( + self, scenario_type: Type[TestScenario] + ) -> List[TestScenarioReport]: + """Find reports for all currently-completed instances of the specified test scenario type.""" + return self._find_test_scenario_reports(scenario_type, self.top_frame) + + def _find_test_scenario_reports( + self, scenario_type: Type[TestScenario], frame: ActionStackFrame + ) -> List[TestScenarioReport]: + results = [] + if ( + frame.report is not None + and "test_scenario" in frame.report + and frame.report.test_scenario is not None + ): + report_scenario_type = get_scenario_type_by_name( + frame.report.test_scenario.scenario_type + ) + if issubclass(report_scenario_type, scenario_type): + results.append(frame.report.test_scenario) + for child in frame.children: + new_results = self._find_test_scenario_reports(scenario_type, child) + results.extend(new_results) + return results + @property def stop_fast(self) -> bool: if ( diff --git a/requirements.txt b/requirements.txt index c22be32137..39eca82fcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,4 +39,4 @@ scipy==1.10.1 shapely==1.7.1 structlog==21.5.0 # deployment_manager termcolor==1.1.0 -uas_standards==3.0.0 +uas_standards==3.1.0 diff --git a/schemas/monitoring/monitorlib/fetch/Query.json b/schemas/monitoring/monitorlib/fetch/Query.json index d69788f3cf..c3474a998e 100644 --- a/schemas/monitoring/monitorlib/fetch/Query.json +++ b/schemas/monitoring/monitorlib/fetch/Query.json @@ -49,7 +49,9 @@ "interuss.automated_testing.flight_planning.v1.GetStatus", "interuss.automated_testing.flight_planning.v1.ClearArea", "interuss.automated_testing.flight_planning.v1.UpsertFlightPlan", - "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" + "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan", + "interuss.automated_testing.rid.v1.observation.getDisplayData", + "interuss.automated_testing.rid.v1.observation.getDetails" ], "type": [ "string",