From 5920de03e75fa6ab91b267dfb54b12a7cf797555 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Tue, 10 Oct 2023 18:22:16 +0200 Subject: [PATCH] Squished commit --- monitoring/prober/infrastructure.py | 2 +- .../astm/netrid/common/dss/isa_validation.py | 339 ++++++++++++++++++ .../scenarios/astm/netrid/common/dss/utils.py | 62 ++++ .../scenarios/astm/netrid/dss_wrapper.py | 100 +++++- .../scenarios/astm/netrid/v19/dss/__init__.py | 1 + .../astm/netrid/v19/dss/isa_validation.md | 95 +++++ .../astm/netrid/v19/dss/isa_validation.py | 8 + .../astm/netrid/v22a/dss/isa_validation.md | 95 +++++ .../astm/netrid/v22a/dss/isa_validation.py | 8 + .../suites/astm/netrid/f3411_19.md | 158 +------- .../astm/netrid/f3411_19/dss_probing.md | 163 +-------- .../astm/netrid/f3411_19/dss_probing.yaml | 38 +- .../suites/interuss/dss/all_tests.md | 158 +------- 13 files changed, 742 insertions(+), 485 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.py diff --git a/monitoring/prober/infrastructure.py b/monitoring/prober/infrastructure.py index c707ca00e4..8e644f3400 100644 --- a/monitoring/prober/infrastructure.py +++ b/monitoring/prober/infrastructure.py @@ -100,7 +100,7 @@ def wrapper_default_scope(*args, **kwargs): resource_type_code_descriptions: Dict[ResourceType, str] = {} -# Next code: 368 +# Next code: 369 def register_resource_type(code: int, description: str) -> ResourceType: """Register that the specified code refers to the described resource. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py new file mode 100644 index 0000000000..8bce90ae6d --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py @@ -0,0 +1,339 @@ +import copy +from typing import Optional, List, Dict, Any + +import arrow +import loguru +import s2sphere +import datetime + +from uas_standards.astm.f3411 import v19, v22a + +from monitoring.monitorlib.fetch import rid as fetch, query_and_describe +from monitoring.monitorlib.fetch.rid import RIDQuery +from monitoring.monitorlib.mutate import rid as mutate +from monitoring.monitorlib.mutate.rid import ISAChange, ChangedISA +from monitoring.monitorlib.rid import RIDVersion +from monitoring.prober.infrastructure import register_resource_type +from monitoring.prober.rid.v1 import common +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource +from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource +from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss import utils +from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + +HUGE_VERTICES: List[s2sphere.LatLng] = [ + s2sphere.LatLng.from_degrees(lng=130, lat=-23), + s2sphere.LatLng.from_degrees(lng=130, lat=-24), + s2sphere.LatLng.from_degrees(lng=132, lat=-24), + s2sphere.LatLng.from_degrees(lng=132, lat=-23), +] + + +class ISAValidation(GenericTestScenario): + """Based on prober/rid/v2/test_isa_validation.py from the legacy prober tool.""" + + ISA_TYPE = register_resource_type(368, "ISA") + + create_isa_path: str + + write_scope: str + + def __init__( + self, + dss: DSSInstanceResource, + id_generator: IDGeneratorResource, + isa: ServiceAreaResource, + ): + super().__init__() + self._dss = ( + dss.dss_instance + ) # TODO: delete once _delete_isa_if_exists updated to use dss_wrapper + self._dss_wrapper = DSSWrapper(self, dss.dss_instance) + self._isa_id = id_generator.id_factory.make_id(ISAValidation.ISA_TYPE) + self._isa_version: Optional[str] = None + self._isa = isa.specification + + now = arrow.utcnow().datetime + self._isa_start_time = self._isa.shifted_time_start(now) + self._isa_end_time = self._isa.shifted_time_end(now) + self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] + + if self._dss.rid_version == RIDVersion.f3411_19: + self.create_isa_path = v19.api.OPERATIONS[ + v19.api.OperationID.CreateSubscription + ].path + self.write_scope = v19.constants.Scope.Write + elif self._dss.rid_version == RIDVersion.f3411_22a: + self.create_isa_path = v22a.api.OPERATIONS[ + v22a.api.OperationID.CreateSubscription + ].path + self.write_scope = v22a.constants.Scope.ServiceProvider + else: + ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + + def run(self): + self.begin_test_scenario() + + self._setup_case() + + self.begin_test_case("ISA Validation") + self.begin_test_step("ISA Validation") + + (create_isa_url, json_body) = self._isa_huge_area_check() + + self._isa_empty_vertices_check() + self._isa_start_time_in_past() + self._isa_start_time_after_time_end() + self._isa_vertices_are_valid() + + self._isa_missing_outline(create_isa_url, json_body) + self._isa_missing_volume(create_isa_url, json_body) + + # TODO Add these: they require omitting things from the request payload + # so we need to bypass the DSSWrapper + # - 'test_isa_missing_outline' + # - 'test_isa_missing_volume' + # - 'test_isa_missing_extents' + + self.end_test_step() + self.end_test_case() + self.end_test_scenario() + + def _setup_case(self): + self.begin_test_case("Setup") + + def _ensure_clean_workspace_step(): + self.begin_test_step("Ensure clean workspace") + + self._delete_isa_if_exists() + + self.end_test_step() + + _ensure_clean_workspace_step() + + self.end_test_case() + + def _delete_isa_if_exists(self): + utils.delete_isa_if_exists( + self, + isa_id=self._isa_id, + rid_version=self._dss.rid_version, + session=self._dss.client, + server_id=self._dss_wrapper.participant_id, + ) + + def _isa_huge_area_check(self) -> (str, Dict[str, Any]): + """Returns the request's URL and json payload for subsequently re-using it. + + It is of the form: + + "extents": { + "volume": { + "outline_polygon": { + "vertices": [], + }, + "altitude_lower": 20.0, + "altitude_upper": 400.0, + }, + "time_start": , + "time_end": , + }, + "uss_base_url": , + """ + + with self.check("ISA huge area", [self._dss_wrapper.participant_id]) as check: + q = self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={400}, + area_vertices=HUGE_VERTICES, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + ) + + return q.dss_query.query.request.url, q.dss_query.query.request.json + + def _isa_empty_vertices_check(self): + + with self.check( + "ISA empty vertices", [self._dss_wrapper.participant_id] + ) as check: + self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={400}, + area_vertices=[], + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + ) + + def _isa_start_time_in_past(self): + time_start = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) + time_end = time_start + datetime.timedelta(minutes=60) + + with self.check( + "ISA start time in the past", [self._dss_wrapper.participant_id] + ) as check: + self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={400}, + area_vertices=self._isa_area, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=time_start, + end_time=time_end, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + ) + + def _isa_start_time_after_time_end(self): + with self.check( + "ISA start time after end time", [self._dss_wrapper.participant_id] + ) as check: + self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={400}, + area_vertices=self._isa_area, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa.time_end.datetime, + end_time=self._isa.time_start.datetime, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + ) + + def _isa_vertices_are_valid(self): + INVALID_VERTICES: List[s2sphere.LatLng] = [ + s2sphere.LatLng.from_degrees(lat=130, lng=-23), + s2sphere.LatLng.from_degrees(lat=130, lng=-24), + s2sphere.LatLng.from_degrees(lat=132, lng=-24), + s2sphere.LatLng.from_degrees(lat=132, lng=-23), + ] + + with self.check( + "ISA vertices are valid", [self._dss_wrapper.participant_id] + ) as check: + self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={400}, + area_vertices=INVALID_VERTICES, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa.time_start.datetime, + end_time=self._isa.time_end.datetime, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + ) + + def _isa_missing_outline(self, create_isa_url: str, json_body: Dict[str, Any]): + payload = copy.deepcopy(json_body) + if self._dss.rid_version == RIDVersion.f3411_19: + del payload["extents"]["spatial_volume"]["footprint"] + elif self._dss.rid_version == RIDVersion.f3411_22a: + del payload["extents"]["volume"]["outline_polygon"] + + with self.check( + "ISA missing outline", [self._dss_wrapper.participant_id] + ) as check: + q = query_and_describe( + client=self._dss.client, + verb="PUT", + url=create_isa_url, + scope=v19.constants.Scope.Write, + json=payload, + ) + if self._dss.rid_version == RIDVersion.f3411_19: + rid_query = ChangedISA(v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + rid_query = ChangedISA(v22a_query=q) + else: + raise ValueError(f"Unknown RID version: {self._dss.rid_version}") + + self._dss_wrapper._handle_query_result( + check=check, + q=rid_query, + fail_msg="ISA Creation with missing outline has unexpected result code", + required_status_code={400}, + severity=Severity.High, + ) + + def _isa_missing_volume(self, create_isa_url: str, json_body: Dict[str, Any]): + payload = copy.deepcopy(json_body) + if self._dss.rid_version == RIDVersion.f3411_19: + del payload["extents"]["spatial_volume"] + elif self._dss.rid_version == RIDVersion.f3411_22a: + del payload["extents"]["volume"] + + with self.check( + "ISA missing volume", [self._dss_wrapper.participant_id] + ) as check: + q = query_and_describe( + client=self._dss.client, + verb="PUT", + url=create_isa_url, + scope=v19.constants.Scope.Write, + json=payload, + ) + if self._dss.rid_version == RIDVersion.f3411_19: + rid_query = ChangedISA(v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + rid_query = ChangedISA(v22a_query=q) + else: + raise ValueError(f"Unknown RID version: {self._dss.rid_version}") + + self._dss_wrapper._handle_query_result( + check=check, + q=rid_query, + fail_msg="ISA Creation with missing outline has unexpected result code", + required_status_code={400}, + severity=Severity.High, + ) + + def _isa_missing_extents(self, create_isa_url: str, json_body: Dict[str, Any]): + payload = copy.deepcopy(json_body) + del payload["extents"] + + with self.check( + "ISA missing extents", [self._dss_wrapper.participant_id] + ) as check: + q = query_and_describe( + client=self._dss.client, + verb="PUT", + url=create_isa_url, + scope=v19.constants.Scope.Write, + json=payload, + ) + if self._dss.rid_version == RIDVersion.f3411_19: + rid_query = ChangedISA(v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + rid_query = ChangedISA(v22a_query=q) + else: + raise ValueError(f"Unknown RID version: {self._dss.rid_version}") + + self._dss_wrapper._handle_query_result( + check=check, + q=rid_query, + fail_msg="ISA Creation with missing outline has unexpected result code", + required_status_code={400}, + severity=Severity.High, + ) + + def cleanup(self): + self.begin_cleanup() + + self._delete_isa_if_exists() + + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py new file mode 100644 index 0000000000..c108b93f96 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py @@ -0,0 +1,62 @@ +from typing import Optional + +from monitoring.monitorlib.fetch import rid as fetch +from monitoring.monitorlib.mutate import rid as mutate +from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + + +def delete_isa_if_exists( + scenario: GenericTestScenario, + isa_id: str, + rid_version: RIDVersion, + session: UTMClientSession, + server_id: Optional[str] = None, +): + fetched = fetch.isa( + isa_id, + rid_version=rid_version, + session=session, + server_id=server_id, + ) + scenario.record_query(fetched.query) + with scenario.check("Successful ISA query", [server_id]) as check: + if not fetched.success and fetched.status_code != 404: + check.record_failed( + "ISA information could not be retrieved", + Severity.High, + f"{server_id} DSS instance returned {fetched.status_code} when queried for ISA {isa_id}", + query_timestamps=[fetched.query.request.timestamp], + ) + + if fetched.success: + deleted = mutate.delete_isa( + isa_id, + fetched.isa.version, + rid_version, + session, + server_id=server_id, + ) + scenario.record_query(deleted.dss_query.query) + for subscriber_id, notification in deleted.notifications.items(): + scenario.record_query(notification.query) + with scenario.check("Removed pre-existing ISA", [server_id]) as check: + if not deleted.dss_query.success: + check.record_failed( + "Could not delete pre-existing ISA", + Severity.High, + f"Attempting to delete ISA {isa_id} from the {server_id} DSS returned error {deleted.dss_query.status_code}", + query_timestamps=[deleted.dss_query.query.request.timestamp], + ) + for subscriber_url, notification in deleted.notifications.items(): + with scenario.check("Notified subscriber", [subscriber_url]) as check: + # TODO: Find a better way to identify a subscriber who couldn't be notified + if not notification.success: + check.record_failed( + "Could not notify ISA subscriber", + Severity.Medium, + f"Attempting to notify subscriber for ISA {isa_id} at {subscriber_url} resulted in {notification.status_code}", + query_timestamps=[notification.query.request.timestamp], + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index dd8b8cbb0b..6c5dd91570 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -1,10 +1,17 @@ import datetime import s2sphere -from typing import Optional, List, Set +from typing import Optional, List, Set, Dict, Any + +from implicitdict import StringBasedDateTime from monitoring.monitorlib import schema_validation -from monitoring.monitorlib.fetch import QueryError +from monitoring.monitorlib.fetch import ( + QueryError, + Query, + RequestDescription, + ResponseDescription, +) from monitoring.monitorlib.fetch.rid import ( FetchedSubscription, FetchedSubscriptions, @@ -15,6 +22,7 @@ from monitoring.monitorlib.mutate import rid as mutate from monitoring.monitorlib.fetch import rid as fetch from monitoring.monitorlib.mutate.rid import ISAChange, ChangedSubscription +from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance from monitoring.uss_qualifier.scenarios.scenario import ( @@ -250,6 +258,43 @@ def get_isa_expect_response_code( return isa + def put_isa_expect_response_code( + self, + check: PendingCheck, + expected_error_codes: Set[int], + area_vertices: List[s2sphere.LatLng], + alt_lo: float, + alt_hi: float, + start_time: datetime.datetime, + end_time: datetime.datetime, + uss_base_url: str, + isa_id: str, + isa_version: Optional[str] = None, + ) -> ISAChange: + mutated_isa = mutate.put_isa( + area_vertices=area_vertices, + alt_lo=alt_lo, + alt_hi=alt_hi, + start_time=start_time, + end_time=end_time, + uss_base_url=uss_base_url, + isa_id=isa_id, + isa_version=isa_version, + rid_version=self._dss.rid_version, + utm_client=self._dss.client, + server_id=self._dss.participant_id, + ) + + self._handle_query_result( + check=check, + q=mutated_isa.dss_query, + fail_msg="ISA Put succeeded when expecting a failure", + required_status_code=expected_error_codes, + severity=Severity.High, + fail_details=f"The submitted query was expected to fail. Payload: {mutated_isa.dss_query.query.request.json}", + ) + return mutated_isa + def put_isa( self, main_check: PendingCheck, @@ -846,3 +891,54 @@ def cleanup_sub( raise RuntimeError( "DSS query was not successful, but a High Severity issue didn't interrupt execution" ) + + def raw_request_with_expected_code( + self, + check: PendingCheck, + method: str, + url_path: str, + json: Dict[str, Any], + expected_error_codes: Set[int], + fail_msg: str, + ) -> RIDQuery: + """For passing raw requests to the underlying client. + Mostly useful for sending malformed requests when testing validations. + """ + + req_descr = RequestDescription( + method=method, + url=url_path, + json=json, + timestamp=datetime.datetime.utcnow(), + ) + + resp = self._dss.client.put(url_path, json=json) + + q = Query( + request=req_descr, + response=ResponseDescription( + code=resp.status_code, + json=resp.json(), + body=resp.content, + headers=resp.headers, + reported=StringBasedDateTime(datetime.datetime.utcnow()), + ), + ) + self._scenario.record_query(q) + + if self._dss.rid_version == RIDVersion.f3411_19: + rid_query = RIDQuery(v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + rid_query = RIDQuery(v22a_query=q) + else: + raise ValueError(f"Unknown RID version: {self._dss.rid_version}") + + self._handle_query_result( + check, + rid_query, + fail_msg, + expected_error_codes, + Severity.Medium, + ) + + return rid_query diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py index a3a46b4509..3a58386869 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py @@ -1,3 +1,4 @@ from .isa_simple import ISASimple +from .isa_validation import ISAValidation from .subscription_validation import SubscriptionValidation from .crdb_access import CRDBAccess diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md new file mode 100644 index 0000000000..78864cb151 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md @@ -0,0 +1,95 @@ +# ASTM NetRID DSS: Submitted ISA Validations test scenario + +## Overview + +Perform basic operations on a single DSS instance to create an ISA and query it during its time of applicability and +after its time of applicability. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing an ISA to be created. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates an ISA with a known ID. This step ensures that ISA does not exist before the start of the main +part of the test. + +#### Successful ISA query check + +**[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0710](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. + + +## ISA Validation test case + +### ISA Validation test step + +#### ISA huge area check + +Attempting to put a too large ISA should result in a 400. + +#### ISA empty vertices check + +An ISA with missing vertices should not result in a successful submission. + +#### ISA start time in the past check + +The DSS must reject ISAs with start times in the past. + +#### ISA start time after end time check + +The DSS must reject ISAs for which the start time is after the end time. + +#### ISA vertices are valid check + +The DSS must reject ISAs with invalid vertices, such as vertices that have latitude or longitude outside meaningful ranges. + +#### ISA missing outline check + +If the outline polygon is missing from the `extents.volume.outline` field in the payload of the ISA creation request, +the DSS is expected to reject the request. + +#### ISA missing volume check + +If the outline polygon is missing from the `extents.volume` field in the payload of the ISA creation request, +the DSS is expected to reject the request. + +#### ISA missing extents check + +If the extents field is missing in the payload of the ISA creation request, +the DSS is expected to reject the request. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. + +### Successful ISA query check + +**[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +### Removed pre-existing ISA check + +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +### Notified subscriber check + +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.py new file mode 100644 index 0000000000..26e070dee1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.isa_validation import ( + ISAValidation as CommonISAValidation, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class ISAValidation(TestScenario, CommonISAValidation): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md new file mode 100644 index 0000000000..5baeb14e5b --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md @@ -0,0 +1,95 @@ +# ASTM NetRID DSS: Submitted ISA Validations test scenario + +## Overview + +Perform basic operations on a single DSS instance to create an ISA and query it during its time of applicability and +after its time of applicability. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing an ISA to be created. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates an ISA with a known ID. This step ensures that ISA does not exist before the start of the main +part of the test. + +#### Successful ISA query check + +**[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0710](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. + + +## ISA Validation test case + +### ISA Validation test step + +#### ISA huge area check + +Attempting to put a too large ISA should result in a 400. + +#### ISA empty vertices check + +An ISA with missing vertices should not result in a successful submission. + +#### ISA start time in the past check + +The DSS must reject ISAs with start times in the past. + +#### ISA start time after end time check + +The DSS must reject ISAs for which the start time is after the end time. + +#### ISA vertices are valid check + +The DSS must reject ISAs with invalid vertices, such as vertices that have latitude or longitude outside meaningful ranges. + +#### ISA missing outline check + +If the outline polygon is missing from the `extents.volume.outline` field in the payload of the ISA creation request, +the DSS is expected to reject the request. + +#### ISA missing volume check + +If the outline polygon is missing from the `extents.volume` field in the payload of the ISA creation request, +the DSS is expected to reject the request. + +#### ISA missing extents check + +If the extents field is missing in the payload of the ISA creation request, +the DSS is expected to reject the request. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. + +### Successful ISA query check + +**[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +### Removed pre-existing ISA check + +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +### Notified subscriber check + +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.py new file mode 100644 index 0000000000..26e070dee1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.isa_validation import ( + ISAValidation as CommonISAValidation, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class ISAValidation(TestScenario, CommonISAValidation): + pass diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 4a00bf06f7..de888e5477 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,160 +21,10 @@ Checked in - astm
.f3411
.v19 - A2-6-1,1a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,2a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,2b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3c - Implemented + TODO - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3d - Implemented + TODO - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,4a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,4b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,5 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,6 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - + astm
.f3411
.v19 DSS0030 Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior - - - DSS0050 - Implemented - ASTM NetRID DSS: Subscription Validation - - - DSS0060 - Implemented - ASTM NetRID DSS: Subscription Validation - - - DSS0070 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0110 - TODO - ASTM NetRID DSS: Direct CRDB access - - - DSS0120 - TODO - ASTM NetRID DSS: Direct CRDB access - - - DSS0130 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,f - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0210 - Implemented - ASTM F3411-19 NetRID DSS interoperability + ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID nominal behavior NET0030 @@ -274,12 +124,12 @@ NET0710 Implemented - ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior + ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID nominal behavior NET0730 Implemented - ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Submitted ISA Validations interuss
.automated_testing
.rid
.injection diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index ad6777da5b..dce3c9cf82 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -4,10 +4,7 @@ ## [Actions](../../../README.md#actions) -1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v19/dss/isa_simple.md) ([`scenarios.astm.netrid.v19.dss.ISASimple`](../../../../scenarios/astm/netrid/v19/dss/isa_simple.py)) -2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) -3. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../../scenarios/astm/netrid/v19/dss_interoperability.py)) -4. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) +1. Scenario: [ASTM NetRID DSS: Submitted ISA Validations](../../../../scenarios/astm/netrid/v19/dss/isa_validation.md) ([`scenarios.astm.netrid.v19.dss.ISAValidation`](../../../../scenarios/astm/netrid/v19/dss/isa_validation.py)) ## [Checked requirements](../../../README.md#checked-requirements) @@ -19,169 +16,19 @@ Checked in - astm
.f3411
.v19 - A2-6-1,1a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,2a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,2b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3c - Implemented + TODO - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3d - Implemented + TODO - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,4a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,4b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,5 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,6 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - + astm
.f3411
.v19 DSS0030 Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation - - - DSS0050 - Implemented - ASTM NetRID DSS: Subscription Validation - - - DSS0060 - Implemented - ASTM NetRID DSS: Subscription Validation - - - DSS0070 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0110 - TODO - ASTM NetRID DSS: Direct CRDB access - - - DSS0120 - TODO - ASTM NetRID DSS: Direct CRDB access - - - DSS0130 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,f - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0210 - Implemented - ASTM F3411-19 NetRID DSS interoperability + ASTM NetRID DSS: Submitted ISA Validations NET0710 Implemented - ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Submitted ISA Validations NET0730 Implemented - ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Submitted ISA Validations diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml index 0fc8819078..629ee2b9a1 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml @@ -5,24 +5,30 @@ resources: id_generator: resources.interuss.IDGeneratorResource isa: resources.netrid.ServiceAreaResource actions: +# - test_scenario: +# scenario_type: scenarios.astm.netrid.v19.dss.ISASimple +# resources: +# dss: dss +# id_generator: id_generator +# isa: isa - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.ISASimple + scenario_type: scenarios.astm.netrid.v19.dss.ISAValidation resources: dss: dss id_generator: id_generator isa: isa - - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionValidation - resources: - dss: dss - id_generator: id_generator - isa: isa - - test_scenario: - scenario_type: scenarios.astm.netrid.v19.DSSInteroperability - resources: - primary_dss_instance: dss - all_dss_instances: all_dss_instances - - test_scenario: - scenario_type: scenarios.astm.netrid.v19.dss.CRDBAccess - resources: {} - on_failure: Continue +# - test_scenario: +# scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionValidation +# resources: +# dss: dss +# id_generator: id_generator +# isa: isa +# - test_scenario: +# scenario_type: scenarios.astm.netrid.v19.DSSInteroperability +# resources: +# primary_dss_instance: dss +# all_dss_instances: all_dss_instances +# - test_scenario: +# scenario_type: scenarios.astm.netrid.v19.dss.CRDBAccess +# resources: {} +# on_failure: Continue diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md index 43e85d6b57..44b9f62520 100644 --- a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md +++ b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md @@ -19,170 +19,20 @@ Checked in - astm
.f3411
.v19 - A2-6-1,1a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,1d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,2a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,2b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3c - Implemented + TODO - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,3d - Implemented + TODO - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,4a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,4b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,5 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - A2-6-1,6 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - + astm
.f3411
.v19 DSS0030 Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation - - - DSS0050 - Implemented - ASTM NetRID DSS: Subscription Validation - - - DSS0060 - Implemented - ASTM NetRID DSS: Subscription Validation - - - DSS0070 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0110 - TODO - ASTM NetRID DSS: Direct CRDB access - - - DSS0120 - TODO - ASTM NetRID DSS: Direct CRDB access - - - DSS0130 - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,2,f - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,a - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,b - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,c - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0130,3,d - Implemented - ASTM F3411-19 NetRID DSS interoperability - - - DSS0210 - Implemented - ASTM F3411-19 NetRID DSS interoperability + ASTM NetRID DSS: Submitted ISA Validations NET0710 Implemented - ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Submitted ISA Validations NET0730 Implemented - ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Submitted ISA Validations astm
.f3411
.v22a