diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index be048f2965..8d3867af11 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -851,19 +851,23 @@ def has_different_content_than(self, other: Any) -> bool: def isas( area: List[s2sphere.LatLng], - start_time: datetime.datetime, - end_time: datetime.datetime, + start_time: Optional[datetime.datetime], + end_time: Optional[datetime.datetime], rid_version: RIDVersion, session: UTMClientSession, dss_base_url: str = "", server_id: Optional[str] = None, ) -> FetchedISAs: - t0 = rid_version.format_time(start_time) - t1 = rid_version.format_time(end_time) + url_time_params = "" + if start_time is not None: + url_time_params += f"&earliest_time={rid_version.format_time(start_time)}" + if end_time is not None: + url_time_params += f"&latest_time={rid_version.format_time(end_time)}" + if rid_version == RIDVersion.f3411_19: op = v19.api.OPERATIONS[v19.api.OperationID.SearchIdentificationServiceAreas] area = rid_v1.geo_polygon_string_from_s2(area) - url = f"{dss_base_url}{op.path}?area={area}&earliest_time={t0}&latest_time={t1}" + url = f"{dss_base_url}{op.path}?area={area}{url_time_params}" return FetchedISAs( v19_query=fetch.query_and_describe( session, @@ -876,7 +880,7 @@ def isas( elif rid_version == RIDVersion.f3411_22a: op = v22a.api.OPERATIONS[v22a.api.OperationID.SearchIdentificationServiceAreas] area = rid_v2.geo_polygon_string_from_s2(area) - url = f"{dss_base_url}{op.path}?area={area}&earliest_time={t0}&latest_time={t1}" + url = f"{dss_base_url}{op.path}?area={area}{url_time_params}" return FetchedISAs( v22a_query=fetch.query_and_describe( session, diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index ce1509035d..5aa9f4ff9e 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -48,6 +48,15 @@ def openapi_flight_details_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") + @property + def openapi_search_isas_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.SearchIdentificationServiceAreasResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.SearchIdentificationServiceAreasResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + @property def openapi_put_isa_response_path(self) -> str: if self == RIDVersion.f3411_19: @@ -57,6 +66,15 @@ def openapi_put_isa_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") + @property + def openapi_delete_isa_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.DeleteIdentificationServiceAreaResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.DeleteIdentificationServiceAreaResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + @property def realtime_period(self) -> timedelta: if self == RIDVersion.f3411_19: diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index 29346ed7cb..9a7492f897 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -16,18 +16,30 @@ class F3411_19(str, Enum): OpenAPIPath = "interfaces/rid/v1/remoteid/augmented.yaml" GetFlightsResponse = "components.schemas.GetFlightsResponse" GetFlightDetailsResponse = "components.schemas.GetFlightDetailsResponse" + SearchIdentificationServiceAreasResponse = ( + "components.schemas.SearchIdentificationServiceAreasResponse" + ) PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) + DeleteIdentificationServiceAreaResponse = ( + "components.schemas.DeleteIdentificationServiceAreaResponse" + ) class F3411_22a(str, Enum): OpenAPIPath = "interfaces/rid/v2/remoteid/updated.yaml" GetFlightsResponse = "components.schemas.GetFlightsResponse" GetFlightDetailsResponse = "components.schemas.GetFlightDetailsResponse" + SearchIdentificationServiceAreasResponse = ( + "components.schemas.SearchIdentificationServiceAreasResponse" + ) PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) + DeleteIdentificationServiceAreaResponse = ( + "components.schemas.DeleteIdentificationServiceAreaResponse" + ) class F3548_21(str, Enum): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index 06aff7fe90..24f43a1a26 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -1,6 +1,8 @@ -from typing import Optional +from typing import Optional, List import arrow +import s2sphere +import datetime from monitoring.monitorlib.fetch import rid as fetch from monitoring.monitorlib.mutate import rid as mutate @@ -12,6 +14,13 @@ from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario +HUGE_VERTICES: List[s2sphere.LatLng] = [ + s2sphere.LatLng.from_degrees(lng=130, lat=-23), + s2sphere.LatLng.from_degrees(lng=130, lat=-24), + s2sphere.LatLng.from_degrees(lng=132, lat=-24), + s2sphere.LatLng.from_degrees(lng=132, lat=-23), +] + class ISASimple(GenericTestScenario): """Based on prober/rid/v2/test_isa_simple.py from the legacy prober tool.""" @@ -141,7 +150,7 @@ def _create_and_check_isa_case(self): def _create_isa_step(): self.begin_test_step("Create ISA") - with self.check("ISA created", [self._dss.participant_id]) as check: + with self.check("ISA created", [self._dss_wrapper.participant_id]) as check: isa_change = self._dss_wrapper.put_isa( main_check=check, area_vertices=self._isa_area, @@ -166,27 +175,310 @@ def _create_isa_step(): def _update_and_search_isa_case(self): self.begin_test_case("Update and search ISA") - # TODO: Update ISA - # TODO: Get ISA by ID - # TODO: Search with invalid params - # TODO: Search by earliest time (included) - # TODO: Search by earliest time (excluded) - # TODO: Search by latest time (included) - # TODO: Search by latest time (excluded) - # TODO: Search by area only - # TODO: Search by huge area + def _update_isa_step(): + self.begin_test_step("Update ISA") + + self._isa_end_time = self._isa_end_time + datetime.timedelta(seconds=1) + with self.check("ISA updated", [self._dss_wrapper.participant_id]) as check: + mutated_isa = self._dss_wrapper.put_isa( + check, + area_vertices=self._isa_area, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=self._isa_version, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + ) + self._isa_version = mutated_isa.dss_query.isa.version + + self.end_test_step() + + _update_isa_step() + + self._get_isa_by_id_step() + + def _search_earliest_incl_step(): + self.begin_test_step("Search by earliest time (included)") + + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + earliest = self._isa_end_time - datetime.timedelta(minutes=1) + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + start_time=earliest, + ) + + with self.check( + "ISA returned by search", [self._dss_wrapper.participant_id] + ) as check: + if self._isa_id not in isas.isas.keys(): + check.record_failed( + f"ISAs search did not return expected ISA {self._isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} from time {earliest} returned ISAs {isas.isas.keys()}", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + self.end_test_step() + + _search_earliest_incl_step() + + def _search_earliest_excl_step(): + self.begin_test_step("Search by earliest time (excluded)") + + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + earliest = self._isa_end_time + datetime.timedelta(minutes=1) + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + start_time=earliest, + ) + + with self.check( + "ISA not returned by search", [self._dss_wrapper.participant_id] + ) as check: + if self._isa_id in isas.isas.keys(): + check.record_failed( + f"ISAs search returned unexpected ISA {self._isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} from time {earliest} returned ISAs {isas.isas.keys()}", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + self.end_test_step() + + _search_earliest_excl_step() + + def _search_latest_incl_step(): + self.begin_test_step("Search by latest time (included)") + + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + latest = self._isa_start_time + datetime.timedelta(minutes=1) + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + end_time=latest, + ) + + with self.check( + "ISA returned by search", [self._dss_wrapper.participant_id] + ) as check: + if self._isa_id not in isas.isas.keys(): + check.record_failed( + f"ISAs search did not return expected ISA {self._isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} to time {latest} returned ISAs {isas.isas.keys()}", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + self.end_test_step() + + _search_latest_incl_step() + + def _search_latest_excl_step(): + self.begin_test_step("Search by latest time (excluded)") + + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + latest = self._isa_start_time - datetime.timedelta(minutes=1) + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + end_time=latest, + ) + + with self.check( + "ISA not returned by search", [self._dss_wrapper.participant_id] + ) as check: + if self._isa_id in isas.isas.keys(): + check.record_failed( + f"ISAs search returned unexpected ISA {self._isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} to time {latest} returned ISAs {isas.isas.keys()}", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + self.end_test_step() + + _search_latest_excl_step() + + def _search_area_only_step(): + self.begin_test_step("Search by area only") + + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + ) + + with self.check( + "ISA returned by search", [self._dss_wrapper.participant_id] + ) as check: + if self._isa_id not in isas.isas.keys(): + check.record_failed( + f"ISAs search did not return expected ISA {self._isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()}", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + self.end_test_step() + + _search_area_only_step() + + def _search_invalid_params_step(): + self.begin_test_step("Search with invalid params") + + with self.check( + "Search request rejected", [self._dss_wrapper.participant_id] + ) as check: + _ = self._dss_wrapper.search_isas_expect_response_code( + check, + expected_error_codes={400}, + area=[], + ) + + self.end_test_step() + + _search_invalid_params_step() + + def _search_huge_area_step(): + self.begin_test_step("Search by huge area") + + with self.check( + "Search request rejected", [self._dss_wrapper.participant_id] + ) as check: + _ = self._dss_wrapper.search_isas_expect_response_code( + check, + expected_error_codes={413}, + area=HUGE_VERTICES, + ) + + self.end_test_step() + + _search_huge_area_step() + + def _search_isa_loop_step(): + self.begin_test_step("Search ISA with loop") + + with self.check( + "Search request rejected", [self._dss_wrapper.participant_id] + ) as check: + search_area_loop = self._isa_area.copy() + search_area_loop.append(search_area_loop[0]) + _ = self._dss_wrapper.search_isas_expect_response_code( + check, + expected_error_codes={400}, + area=search_area_loop, + ) + + self.end_test_step() + + _search_isa_loop_step() self.end_test_case() def _delete_isa_case(self): self.begin_test_case("Delete ISA") - # TODO: Delete with wrong version - # TODO: Delete with empty version - # TODO: Delete ISA - # TODO: Get ISA by ID - # TODO: Search ISA - # TODO: Search ISA with loop + def _delete_wrong_version_step(): + self.begin_test_step("Delete with wrong version") + + with self.check( + "Delete request rejected", [self._dss_wrapper.participant_id] + ) as check: + _ = self._dss_wrapper.del_isa_expect_response_code( + check, + expected_error_codes={409}, + isa_id=self._isa_id, + isa_version=self._isa_version[1:-1], + ) + + self.end_test_step() + + _delete_wrong_version_step() + + def _delete_empty_version_step(): + self.begin_test_step("Delete with empty version") + + with self.check( + "Delete request rejected", [self._dss_wrapper.participant_id] + ) as check: + _ = self._dss_wrapper.del_isa_expect_response_code( + check, + expected_error_codes={400}, + isa_id=self._isa_id, + isa_version="", + ) + + self.end_test_step() + + _delete_empty_version_step() + + def _delete_step(): + self.begin_test_step("Delete ISA") + + with self.check("ISA deleted", [self._dss_wrapper.participant_id]) as check: + _ = self._dss_wrapper.del_isa( + check, isa_id=self._isa_id, isa_version=self._isa_version + ) + + self.end_test_step() + + _delete_step() + + def _get_deleted_isa_by_id_step(): + self.begin_test_step("Get deleted ISA by ID") + + with self.check( + "ISA not found", [self._dss_wrapper.participant_id] + ) as check: + _ = self._dss_wrapper.get_isa_expect_response_code( + check, + expected_error_codes={404}, + isa_id=self._isa_id, + ) + + self.end_test_step() + + _get_deleted_isa_by_id_step() + + def _search_isa_step(): + self.begin_test_step("Search ISA") + + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + ) + + with self.check( + "ISA not returned by search", [self._dss_wrapper.participant_id] + ) as check: + if self._isa_id in isas.isas.keys(): + check.record_failed( + f"ISAs search returned deleted ISA {self._isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()}", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + self.end_test_step() + + _search_isa_step() self.end_test_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 32b9dca914..dd8b8cbb0b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -97,6 +97,90 @@ def _handle_query_result( query_timestamps=[q.query.request.timestamp], ) + def search_isas( + self, + main_check: PendingCheck, + area: List[s2sphere.LatLng], + start_time: Optional[datetime.datetime] = None, + end_time: Optional[datetime.datetime] = None, + ) -> FetchedISAs: + """Search for ISAs at the DSS. + + Query failure will fail the provided main check. If the query is successful, the sub-checks of the test step + described in '[v19|v22a]/dss/test_steps/search_isas.md' are performed. Some of those might fail the main check. + + :return: the DSS response + """ + + isas = fetch.isas( + area=area, + start_time=start_time, + end_time=end_time, + rid_version=self._dss.rid_version, + session=self._dss.client, + server_id=self._dss.participant_id, + ) + self._handle_query_result( + main_check, + isas, + f"Failed to search ISAs in {area} from {start_time} to {end_time}", + ) + + dss_id = [self._dss.participant_id] + t_dss = isas.query.request.timestamp + + with self._scenario.check("ISAs search response format", dss_id) as sub_check: + errors = schema_validation.validate( + self._dss.rid_version.openapi_path, + self._dss.rid_version.openapi_search_isas_response_path, + isas.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + sub_check.record_failed( + "Search ISA response format was invalid", + Severity.Medium, + "Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) + + return isas + + def search_isas_expect_response_code( + self, + main_check: PendingCheck, + expected_error_codes: Set[int], + area: List[s2sphere.LatLng], + start_time: Optional[datetime.datetime] = None, + end_time: Optional[datetime.datetime] = None, + ) -> FetchedISAs: + """Attempt to search for ISAs at the DSS, and expect the specified HTTP response code. + + A check fail is considered of high severity and as such will raise a ScenarioCannotContinueError. + + :return: the DSS response + """ + + isas = fetch.isas( + area=area, + start_time=start_time, + end_time=end_time, + rid_version=self._dss.rid_version, + session=self._dss.client, + server_id=self._dss.participant_id, + ) + + self._handle_query_result( + check=main_check, + q=isas, + required_status_code=expected_error_codes, + fail_msg=f"Searching for ISAs resulted in an HTTP code not in {expected_error_codes}", + fail_details=f"Search area: {area}; from {start_time} to {end_time}", + ) + + return isas + def get_isa( self, check: PendingCheck, @@ -136,6 +220,36 @@ def get_isa( "DSS query was not successful, but a High Severity issue didn't interrupt execution" ) + def get_isa_expect_response_code( + self, + check: PendingCheck, + expected_error_codes: Set[int], + isa_id: str, + ) -> FetchedISA: + """Attempt to fetch an ISA at the DSS, and expect the specified HTTP response code. + + A check fail is considered of high severity and as such will raise a ScenarioCannotContinueError. + + :return: the DSS response + """ + + isa = fetch.isa( + isa_id=isa_id, + rid_version=self._dss.rid_version, + session=self._dss.client, + server_id=self._dss.participant_id, + ) + + self._handle_query_result( + check=check, + q=isa, + required_status_code=expected_error_codes, + fail_msg=f"Fetching ISA {isa_id} resulted in an HTTP code not in {expected_error_codes}", + fail_details=f"ISA: ID {isa_id}", + ) + + return isa + def put_isa( self, main_check: PendingCheck, @@ -281,46 +395,121 @@ def _fail_sub_check( def del_isa( self, - check: PendingCheck, + main_check: PendingCheck, isa_id: str, isa_version: str, ) -> ISAChange: """Delete an ISA at the DSS. - A check fail is considered of high severity and as such will raise a ScenarioCannotContinueError. + + Query failure will fail the provided main check. If the query is successful, the sub-checks of the test step + described in '[v19|v22a]/dss/test_steps/delete_isa.md' are performed. Some of those might fail the main check. :return: the DSS response """ - try: - del_isa = mutate.delete_isa( - isa_id=isa_id, - isa_version=isa_version, - rid_version=self._dss.rid_version, - utm_client=self._dss.client, - server_id=self._dss.participant_id, + del_isa = mutate.delete_isa( + isa_id=isa_id, + isa_version=isa_version, + rid_version=self._dss.rid_version, + utm_client=self._dss.client, + server_id=self._dss.participant_id, + ) + self._handle_query_result( + main_check, del_isa.dss_query, f"Failed to delete ISA {isa_id}" + ) + for notification_query in del_isa.notifications.values(): + self._scenario.record_query(notification_query.query) + + dss_id = [self._dss.participant_id] + t_dss = del_isa.dss_query.query.request.timestamp + dss_isa = del_isa.dss_query.isa + + # sub-checks that do not fail the main check + with self._scenario.check("ISA response format", dss_id) as sub_check: + errors = schema_validation.validate( + self._dss.rid_version.openapi_path, + self._dss.rid_version.openapi_delete_isa_response_path, + del_isa.dss_query.query.response.json, ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + sub_check.record_failed( + "Delete ISA response format was invalid", + Severity.Medium, + "Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) - self._handle_query_result( - check, del_isa.dss_query, f"Failed to delete ISA {isa_id}" + # sub-checks that fail the main check + def _fail_sub_check( + _sub_check: PendingCheck, _summary: str, _details: str + ) -> None: + """Fails with Medium severity the sub_check and with High severity the main check.""" + + _sub_check.record_failed( + summary=_summary, + severity=Severity.Medium, + details=_details, + query_timestamps=[t_dss], + ) + main_check.record_failed( + summary=f"Delete ISA request succeeded, but the DSS response is not valid: {_summary}", + severity=Severity.High, + details=_details, + query_timestamps=[t_dss], ) - if isa_version != del_isa.dss_query.isa.version: - check.record_failed( - summary=f"Deleted ISA did not match", - severity=Severity.High, - participants=[self._dss.participant_id], - details=f"DSS reported deletion of version {isa_version} while expecting {del_isa.dss_query.isa.version}", - query_timestamps=[del_isa.dss_query.query.request.timestamp], + with self._scenario.check("ISA ID matches", dss_id) as sub_check: + if isa_id != dss_isa.id: + _fail_sub_check( + sub_check, + "Deleted ISA ID did not match", + f"Expected ISA ID {isa_id} but got {dss_isa.id}", ) - else: - return del_isa - except QueryError as e: - self._handle_query_error(check, e) - raise RuntimeError( - "DSS query was not successful, but a High Severity issue didn't interrupt execution" + with self._scenario.check("ISA version matches", dss_id) as sub_check: + if dss_isa.version != isa_version: + _fail_sub_check( + sub_check, + "Deleted ISA version did not match", + f"Expected ISA version {isa_version} but got {dss_isa.version}", + ) + + return del_isa + + def del_isa_expect_response_code( + self, + main_check: PendingCheck, + expected_error_codes: Set[int], + isa_id: str, + isa_version: str, + ) -> ISAChange: + """Attempt to delete an ISA at the DSS, and expect the specified HTTP response code. + + A check fail is considered of high severity and as such will raise a ScenarioCannotContinueError. + + :return: the DSS response + """ + + del_isa = mutate.delete_isa( + isa_id=isa_id, + isa_version=isa_version, + rid_version=self._dss.rid_version, + utm_client=self._dss.client, + server_id=self._dss.participant_id, ) + self._handle_query_result( + check=main_check, + q=del_isa.dss_query, + required_status_code=expected_error_codes, + fail_msg=f"Deleting ISA {isa_id} resulted in an HTTP code not in {expected_error_codes}", + fail_details=f"ISA: ID {isa_id}; version {isa_version}", + ) + + return del_isa + def cleanup_isa( self, check: PendingCheck, diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index a1cce5c6d5..7ba1e3d551 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -65,8 +65,160 @@ The DSS returns the version of the ISA in the response body. If this version do ## Update and search ISA test case +### [Update ISA test step](test_steps/put_isa.md) + +This step attempts to update the configured DSS with the ISA provided as a resource, with a slightly different end time. + +#### ISA updated check + +If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +### Get ISA by ID test step + +This step attempts to retrieve at the DSS the ISA just updated. + +#### Successful ISA query check + +If the ISA cannot be queried, the GET ISA DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +#### ISA version match check + +The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after update, and that no modification of the ISA occurred in the meantime, **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +### [Search by earliest time (included) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that overlaps with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### ISA returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### [Search by earliest time (excluded) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that does not overlap with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### ISA not returned by search check + +The ISA search are parameter cover the resource ISA but the earliest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### [Search by latest time (included) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that overlaps with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### ISA returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### [Search by latest time (excluded) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that does not overlap with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### ISA not returned by search check + +The ISA search are parameter cover the resource ISA but the latest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### [Search by area only test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with only the area of the ISA resource. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### ISA returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### Search with invalid params test step + +This step attempts an ISA search at the DSS with an empty search area. + +#### Search request rejected check + +The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### Search by huge area test step + +This step attempts an ISA search at the DSS with a too large search area. + +#### Search request rejected check + +The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### Search ISA with loop test step + +This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. + +#### Search request rejected check + +The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + + ## Delete ISA test case +### Delete with wrong version test step + +This step attempts an ISA deletion with a wrong version. + +#### Delete request rejected check + +The deletion request contained invalid parameters (wrong version), as such the DSS should reject it with a 409 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### Delete with empty version test step + +This step attempts an ISA deletion with an empty version. + +#### Delete request rejected check + +The deletion request contained invalid parameters (empty version), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### [Delete ISA test step](test_steps/delete_isa.md) + +This step attempts an ISA deletion at the DSS. + +#### ISA deleted check + +If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +### Get deleted ISA by ID test step + +This step attempts to retrieve at the DSS the ISA just deleted. + +#### ISA not found check + +The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +### [Search ISA test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with only the area of the ISA resource. Since it has just been deleted, the ISA should not be returned. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### ISA not returned by search check + +The ISA search are parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + + ## Cleanup The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/delete_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/delete_isa.md new file mode 100644 index 0000000000..dc24be0f0c --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/delete_isa.md @@ -0,0 +1,16 @@ +# Delete ISA test step + +This page describes the content of a common test step where a deletion of an ISA should be successful. +See `DSSWrapper.del_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). + +## ISA response format check + +The API for **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +## ISA ID matches check + +When the ISA is deleted, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. + +## ISA version matches check + +When the ISA is deleted, the DSS returns the version of the ISA in the response body. If this version does not match the version in the resource path, **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** was not implemented correctly and this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md new file mode 100644 index 0000000000..f1cfb958ce --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md @@ -0,0 +1,8 @@ +# Search ISAs test step + +This page describes the content of a common test step where a search for ISAs should be successful. +See `DSSWrapper.search_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). + +## ISAs search response format check + +The API for **[astm.f3411.v19.DSS0030](../../../../../../requirements/astm/f3411/v19.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md index c3af41eadf..1cd0a31eb1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md @@ -228,7 +228,7 @@ Qualitatively proves: ISA creation triggers subscription notification requests **[astm.f3411.v19.A2-6-1,3b](../../../../requirements/astm/f3411/v19.md)** -### S11 test step +### [S11 test step](dss/test_steps/delete_isa.md) Action: USS1@DSS*P*: DELETE ISA[*P*] @@ -284,7 +284,7 @@ TODO: Investigate expected behavior and "404 with proper response" check **[astm.f3411.v19.A2-6-1,3d](../../../../requirements/astm/f3411/v19.md)** -### S15 test step +### [S15 test step](dss/test_steps/delete_isa.md) Action: USS1@DSS*P*: DELETE ISA[*P*] diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index c0fee7ad4b..29ca35daf9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -65,8 +65,160 @@ The DSS returns the version of the ISA in the response body. If this version do ## Update and search ISA test case +### [Update ISA test step](test_steps/put_isa.md) + +This step attempts to update the configured DSS with the ISA provided as a resource, with a slightly different end time. + +#### ISA updated check + +If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +### Get ISA by ID test step + +This step attempts to retrieve at the DSS the ISA just updated. + +#### Successful ISA query check + +If the ISA cannot be queried, the GET ISA DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +#### ISA version match check + +The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after update, and that no modification of the ISA occurred in the meantime, **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +### [Search by earliest time (included) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that overlaps with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### ISA returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Search by earliest time (excluded) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that does not overlap with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### ISA not returned by search check + +The ISA search are parameter cover the resource ISA but the earliest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Search by latest time (included) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that overlaps with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### ISA returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Search by latest time (excluded) test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that does not overlap with the resource ISA. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### ISA not returned by search check + +The ISA search are parameter cover the resource ISA but the latest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Search by area only test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with only the area of the ISA resource. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### ISA returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### Search with invalid params test step + +This step attempts an ISA search at the DSS with an empty search area. + +#### Search request rejected check + +The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### Search by huge area test step + +This step attempts an ISA search at the DSS with a too large search area. + +#### Search request rejected check + +The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### Search ISA with loop test step + +This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. + +#### Search request rejected check + +The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + + ## Delete ISA test case +### Delete with wrong version test step + +This step attempts an ISA deletion with a wrong version. + +#### Delete request rejected check + +The deletion request contained invalid parameters (wrong version), as such the DSS should reject it with a 409 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### Delete with empty version test step + +This step attempts an ISA deletion with an empty version. + +#### Delete request rejected check + +The deletion request contained invalid parameters (empty version), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Delete ISA test step](test_steps/delete_isa.md) + +This step attempts an ISA deletion at the DSS. + +#### ISA deleted check + +If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +### Get deleted ISA by ID test step + +This step attempts to retrieve at the DSS the ISA just deleted. + +#### ISA not found check + +The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +### [Search ISA test step](test_steps/search_isas.md) + +This step attempts an ISA search at the DSS with only the area of the ISA resource. Since it has just been deleted, the ISA should not be returned. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### ISA not returned by search check + +The ISA search are parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + + ## Cleanup The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/delete_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/delete_isa.md new file mode 100644 index 0000000000..85a475879e --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/delete_isa.md @@ -0,0 +1,16 @@ +# Delete ISA test step + +This page describes the content of a common test step where a deletion of an ISA should be successful. +See `DSSWrapper.del_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). + +## ISA response format check + +The API for **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +## ISA ID matches check + +When the ISA is deleted, the DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. + +## ISA version matches check + +When the ISA is deleted, the DSS returns the version of the ISA in the response body. If this version does not match the version in the resource path, **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** was not implemented correctly and this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md new file mode 100644 index 0000000000..f7d5705563 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md @@ -0,0 +1,8 @@ +# Search ISAs test step + +This page describes the content of a common test step where a search for ISAs should be successful. +See `DSSWrapper.search_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). + +## ISAs search response format check + +The API for **[astm.f3411.v22a.DSS0030](../../../../../../requirements/astm/f3411/v22a.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md index 5a4723d639..7b8fd80b7e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md @@ -227,7 +227,7 @@ Qualitatively proves: ISA creation triggers subscription notification requests **[astm.f3411.v22a.A2-6-1,3b](../../../../requirements/astm/f3411/v22a.md)** -### S11 test step +### [S11 test step](dss/test_steps/delete_isa.md) Action: USS1@DSS*P*: DELETE ISA[*P*] @@ -283,7 +283,7 @@ TODO: Investigate expected behavior and "404 with proper response" check **[astm.f3411.v22a.A2-6-1,3d](../../../../requirements/astm/f3411/v22a.md)** -### S15 test step +### [S15 test step](dss/test_steps/delete_isa.md) Action: USS1@DSS*P*: DELETE ISA[*P*]