From b418c994002377ab9711ba49d2e8ded33f730ca5 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Fri, 17 Nov 2023 11:20:55 +0100 Subject: [PATCH] [uss_qualifier] DSS0030 heavy_traffic_concurrent migration --- monitoring/monitorlib/fetch/__init__.py | 15 + monitoring/monitorlib/infrastructure.py | 38 +- monitoring/monitorlib/mutate/rid.py | 100 ++-- monitoring/monitorlib/rid.py | 9 + monitoring/monitorlib/schema_validation.py | 6 + monitoring/prober/infrastructure.py | 2 +- .../common/dss/heavy_traffic_concurrent.py | 449 ++++++++++++++++++ .../scenarios/astm/netrid/common/dss/utils.py | 262 +++++++++- .../scenarios/astm/netrid/v19/dss/__init__.py | 1 + .../v19/dss/heavy_traffic_concurrent.md | 188 ++++++++ .../v19/dss/heavy_traffic_concurrent.py | 8 + .../astm/netrid/v22a/dss/__init__.py | 1 + .../v22a/dss/heavy_traffic_concurrent.md | 188 ++++++++ .../v22a/dss/heavy_traffic_concurrent.py | 8 + .../suites/astm/netrid/f3411_19.md | 10 +- .../astm/netrid/f3411_19/dss_probing.md | 11 +- .../astm/netrid/f3411_19/dss_probing.yaml | 6 + .../suites/astm/netrid/f3411_22a.md | 10 +- .../astm/netrid/f3411_22a/dss_probing.md | 11 +- .../astm/netrid/f3411_22a/dss_probing.yaml | 6 + .../suites/interuss/dss/all_tests.md | 16 +- .../suites/uspace/network_identification.md | 10 +- .../suites/uspace/required_services.md | 10 +- .../mock_uss/client/MockUSSSpecification.json | 7 + 24 files changed, 1293 insertions(+), 79 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.py diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index d69a3eb321..4ad629962c 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -8,6 +8,7 @@ from enum import Enum from urllib.parse import urlparse +import aiohttp import flask from loguru import logger import requests @@ -145,6 +146,20 @@ def describe_response(resp: requests.Response) -> ResponseDescription: return ResponseDescription(**kwargs) +def describe_aiohttp_response( + status: int, headers: Dict, resp_json: Dict, duration: datetime.timedelta +) -> ResponseDescription: + kwargs = { + "code": status, + "headers": headers, + "elapsed_s": duration.total_seconds(), + "reported": StringBasedDateTime(datetime.datetime.utcnow()), + "json": resp_json, + } + + return ResponseDescription(**kwargs) + + def describe_flask_response(resp: flask.Response, elapsed_s: float): headers = {k: v for k, v in resp.headers.items()} kwargs = { diff --git a/monitoring/monitorlib/infrastructure.py b/monitoring/monitorlib/infrastructure.py index 093ef73c28..cf3e30b34a 100644 --- a/monitoring/monitorlib/infrastructure.py +++ b/monitoring/monitorlib/infrastructure.py @@ -3,7 +3,7 @@ import functools from typing import Dict, List, Optional import urllib.parse -from aiohttp import ClientSession +from aiohttp import ClientSession, ClientResponse import jwt import requests @@ -189,19 +189,35 @@ def adjust_request_kwargs(self, url, method, kwargs): kwargs["timeout"] = self.timeout_seconds return kwargs - async def put(self, url, **kwargs): + async def put_with_headers(self, url, **kwargs): url = self._prefix_url + url if "auth" not in kwargs: kwargs = self.adjust_request_kwargs(url, "PUT", kwargs) async with self._client.put(url, **kwargs) as response: - return response.status, await response.json() + return ( + response.status, + {k: v for k, v in response.headers.items()}, + await response.json(), + ) - async def get(self, url, **kwargs): + async def put(self, url, **kwargs): + (status, _, json) = await self.put_with_headers(url, **kwargs) + return status, json + + async def get_with_headers(self, url, **kwargs): url = self._prefix_url + url if "auth" not in kwargs: kwargs = self.adjust_request_kwargs(url, "GET", kwargs) async with self._client.get(url, **kwargs) as response: - return response.status, await response.json() + return ( + response.status, + {k: v for k, v in response.headers.items()}, + await response.json(), + ) + + async def get(self, url, **kwargs): + (status, _, json) = await self.get_with_headers(url, **kwargs) + return status, json async def post(self, url, **kwargs): url = self._prefix_url + url @@ -210,12 +226,20 @@ async def post(self, url, **kwargs): async with self._client.post(url, **kwargs) as response: return response.status, await response.json() - async def delete(self, url, **kwargs): + async def delete_with_headers(self, url, **kwargs): url = self._prefix_url + url if "auth" not in kwargs: kwargs = self.adjust_request_kwargs(url, "DELETE", kwargs) async with self._client.delete(url, **kwargs) as response: - return response.status, await response.json() + return ( + response.status, + {k: v for k, v in response.headers.items()}, + await response.json(), + ) + + async def delete(self, url, **kwargs): + (status, _, json) = await self.delete_with_headers(url, **kwargs) + return status, json def default_scopes(scopes: List[str]): diff --git a/monitoring/monitorlib/mutate/rid.py b/monitoring/monitorlib/mutate/rid.py index 09fbe25cf0..fa2e72bb3d 100644 --- a/monitoring/monitorlib/mutate/rid.py +++ b/monitoring/monitorlib/mutate/rid.py @@ -3,6 +3,7 @@ from implicitdict import ImplicitDict import s2sphere +from uas_standards import Operation from monitoring.monitorlib.fetch.rid import RIDQuery, Subscription, ISA from monitoring.monitorlib.rid import RIDVersion @@ -439,22 +440,19 @@ class ISAChange(ImplicitDict): """Mapping from USS base URL to change notification query""" -def put_isa( +def build_isa_payload( area_vertices: List[s2sphere.LatLng], alt_lo: float, alt_hi: float, start_time: datetime.datetime, end_time: datetime.datetime, uss_base_url: str, - isa_id: str, rid_version: RIDVersion, - utm_client: infrastructure.UTMClientSession, - isa_version: Optional[str] = None, - participant_id: Optional[str] = None, -) -> ISAChange: - mutation = "create" if isa_version is None else "update" +) -> Dict[str, any]: + """Build the payload expected to PUT or UPDATE an ISA on a DSS, + in accordance with the specified rid_version.""" if rid_version == RIDVersion.f3411_19: - body = { + return { "extents": rid_v1.make_volume_4d( area_vertices, alt_lo, @@ -465,25 +463,8 @@ def put_isa( "flights_url": uss_base_url + v19.api.OPERATIONS[v19.api.OperationID.SearchFlights].path, } - if isa_version is None: - op = v19.api.OPERATIONS[v19.api.OperationID.CreateIdentificationServiceArea] - url = op.path.format(id=isa_id) - else: - op = v19.api.OPERATIONS[v19.api.OperationID.UpdateIdentificationServiceArea] - url = op.path.format(id=isa_id, version=isa_version) - dss_response = ChangedISA( - mutation=mutation, - v19_query=fetch.query_and_describe( - utm_client, - op.verb, - url, - json=body, - scope=v19.constants.Scope.Write, - participant_id=participant_id, - ), - ) elif rid_version == RIDVersion.f3411_22a: - body = { + return { "extents": rid_v2.make_volume_4d( area_vertices, alt_lo, @@ -493,16 +474,79 @@ def put_isa( ), "uss_base_url": uss_base_url, } + else: + raise NotImplementedError( + f"Cannot build ISA payload for RID version {rid_version}" + ) + + +def build_isa_url( + rid_version: RIDVersion, isa_id: str, isa_version: Optional[str] = None +) -> (Operation, str): + """Build the required URL to create, get, update or delete an ISA on a DSS, + in accordance with the specified rid_version and isa_version, if it is available. + + Note that for mutations and deletions, isa_version must be provided. + """ + if rid_version == RIDVersion.f3411_19: + if isa_version is None: + op = v19.api.OPERATIONS[v19.api.OperationID.CreateIdentificationServiceArea] + return (op, op.path.format(id=isa_id)) + else: + op = v19.api.OPERATIONS[v19.api.OperationID.UpdateIdentificationServiceArea] + return (op, op.path.format(id=isa_id, version=isa_version)) + elif rid_version == RIDVersion.f3411_22a: if isa_version is None: op = v22a.api.OPERATIONS[ v22a.api.OperationID.CreateIdentificationServiceArea ] - url = op.path.format(id=isa_id) + return (op, op.path.format(id=isa_id)) else: op = v22a.api.OPERATIONS[ v22a.api.OperationID.UpdateIdentificationServiceArea ] - url = op.path.format(id=isa_id, version=isa_version) + return (op, op.path.format(id=isa_id, version=isa_version)) + else: + raise NotImplementedError(f"Cannot build ISA URL for RID version {rid_version}") + + +def put_isa( + area_vertices: List[s2sphere.LatLng], + alt_lo: float, + alt_hi: float, + start_time: datetime.datetime, + end_time: datetime.datetime, + uss_base_url: str, + isa_id: str, + rid_version: RIDVersion, + utm_client: infrastructure.UTMClientSession, + isa_version: Optional[str] = None, + participant_id: Optional[str] = None, +) -> ISAChange: + mutation = "create" if isa_version is None else "update" + body = build_isa_payload( + area_vertices, + alt_lo, + alt_hi, + start_time, + end_time, + uss_base_url, + rid_version, + ) + (op, url) = build_isa_url(rid_version, isa_id, isa_version) + if rid_version == RIDVersion.f3411_19: + dss_response = ChangedISA( + mutation=mutation, + v19_query=fetch.query_and_describe( + utm_client, + op.verb, + url, + json=body, + scope=v19.constants.Scope.Write, + participant_id=participant_id, + ), + ) + elif rid_version == RIDVersion.f3411_22a: dss_response = ChangedISA( mutation=mutation, v22a_query=fetch.query_and_describe( diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index 5aa9f4ff9e..3d2826de41 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -57,6 +57,15 @@ def openapi_search_isas_response_path(self) -> str: else: raise ValueError(f"Unsupported RID version '{self}'") + @property + def openapi_get_isa_response_path(self) -> str: + if self == RIDVersion.f3411_19: + return schema_validation.F3411_19.GetIdentificationServiceAreaResponse + elif self == RIDVersion.f3411_22a: + return schema_validation.F3411_22a.GetIdentificationServiceAreaResponse + else: + raise ValueError(f"Unsupported RID version '{self}'") + @property def openapi_put_isa_response_path(self) -> str: if self == RIDVersion.f3411_19: diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index 9a7492f897..ce0c43786f 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -19,6 +19,9 @@ class F3411_19(str, Enum): SearchIdentificationServiceAreasResponse = ( "components.schemas.SearchIdentificationServiceAreasResponse" ) + GetIdentificationServiceAreaResponse = ( + "components.schemas.GetIdentificationServiceAreaResponse" + ) PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) @@ -34,6 +37,9 @@ class F3411_22a(str, Enum): SearchIdentificationServiceAreasResponse = ( "components.schemas.SearchIdentificationServiceAreasResponse" ) + GetIdentificationServiceAreaResponse = ( + "components.schemas.GetIdentificationServiceAreaResponse" + ) PutIdentificationServiceAreaResponse = ( "components.schemas.PutIdentificationServiceAreaResponse" ) diff --git a/monitoring/prober/infrastructure.py b/monitoring/prober/infrastructure.py index 5a5f145965..d0124b10c6 100644 --- a/monitoring/prober/infrastructure.py +++ b/monitoring/prober/infrastructure.py @@ -100,7 +100,7 @@ def wrapper_default_scope(*args, **kwargs): resource_type_code_descriptions: Dict[ResourceType, str] = {} -# Next code: 373 +# Next code: 374 def register_resource_type(code: int, description: str) -> ResourceType: """Register that the specified code refers to the described resource. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py new file mode 100644 index 0000000000..8dbcd9bb5d --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py @@ -0,0 +1,449 @@ +import asyncio +import typing +from datetime import datetime +from typing import List, Dict + +import arrow +import requests +from uas_standards.astm.f3411 import v19, v22a + +from monitoring.monitorlib.fetch import ( + rid as fetch, + describe_request, + Query, + describe_aiohttp_response, +) +from monitoring.monitorlib.fetch.rid import FetchedISA +from monitoring.monitorlib.infrastructure import AsyncUTMTestSession +from monitoring.monitorlib.mutate import rid as mutate +from monitoring.monitorlib.mutate.rid import ChangedISA +from monitoring.monitorlib.rid import RIDVersion +from monitoring.prober.infrastructure import register_resource_type +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource +from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource +from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.utils import ( + ISAValidator, + delete_isa_if_exists, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario +from monitoring.uss_qualifier.suites.suite import ExecutionContext + +# Semaphore is added to limit the number of simultaneous requests. +# Should we consider making these configurable through the scenario's parameters? +SEMAPHORE = asyncio.Semaphore(20) +THREAD_COUNT = 10 +CREATE_ISAS_COUNT = 100 + + +class HeavyTrafficConcurrent(GenericTestScenario): + """Based on prober/rid/v1/test_isa_simple_heavy_traffic_concurrent.py from the legacy prober tool.""" + + ISA_TYPE = register_resource_type(373, "ISA") + + _isa_ids: List[str] + + _isa_params: Dict[str, any] + + _isa_versions: Dict[str, str] + + _async_session: AsyncUTMTestSession + + def __init__( + self, + dss: DSSInstanceResource, + id_generator: IDGeneratorResource, + isa: ServiceAreaResource, + ): + super().__init__() + self._dss = ( + dss.dss_instance + ) # TODO: delete once _delete_isa_if_exists updated to use dss_wrapper + self._dss_wrapper = DSSWrapper(self, dss.dss_instance) + + self._isa_versions: Dict[str, str] = {} + self._isa = isa.specification + + now = arrow.utcnow().datetime + self._isa_start_time = self._isa.shifted_time_start(now) + self._isa_end_time = self._isa.shifted_time_end(now) + self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] + + # TODO confirm that the auth-adapter can be reused as-is (this seems to work but a confirmation would be good) + # Note that when the test scenario ends prematurely, we may end up with an unclosed session. + self._async_session = AsyncUTMTestSession( + self._dss.base_url, self._dss.client.auth_adapter + ) + + isa_base_id = id_generator.id_factory.make_id(HeavyTrafficConcurrent.ISA_TYPE) + # The base ID ends in 000: we simply increment it to generate the other IDs + self._isa_ids = [f"{isa_base_id[:-3]}{i:03d}" for i in range(CREATE_ISAS_COUNT)] + + # currently all params are the same: + # we could improve the test by having unique parameters per ISA + self._isa_params = dict( + area_vertices=self._isa_area, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + ) + + def run(self, context: ExecutionContext): + self.begin_test_scenario(context) + + self.begin_test_case("Setup") + self.begin_test_step("Ensure clean workspace") + self._delete_isas_if_exists() + self.end_test_step() + self.end_test_case() + + self.begin_test_case("Concurrent Requests") + + self.begin_test_step("Create ISA concurrently") + self._create_isas_concurrent_step() + self.end_test_step() + + self.begin_test_step("Get ISAs concurrently") + self._get_isas_by_id_concurrent_step() + self.end_test_step() + + self.begin_test_step("Search Available ISAs") + self._search_area_step() + self.end_test_step() + + self.begin_test_step("Delete ISAs concurrently") + self._delete_isas() + self.end_test_step() + + self.begin_test_step("Access Deleted ISAs") + self._get_deleted_isas() + self.end_test_step() + + self.begin_test_step("Search Deleted ISAs") + self._search_deleted_isas() + self.end_test_step() + + self.end_test_case() + self.end_test_scenario() + + def _delete_isas_if_exists(self): + """Delete test ISAs if they exist. Done sequentially.""" + for isa_id in self._isa_ids: + delete_isa_if_exists( + self, + isa_id=isa_id, + rid_version=self._dss.rid_version, + session=self._dss.client, + participant_id=self._dss_wrapper.participant_id, + ) + + def _get_isas_by_id_concurrent_step(self): + loop = asyncio.get_event_loop() + results = loop.run_until_complete( + asyncio.gather(*[self._get_isa(isa_id) for isa_id in self._isa_ids]) + ) + + results = typing.cast(Dict[str, FetchedISA], results) + + for _, fetched_isa in results: + self.record_query(fetched_isa.query) + + with self.check( + "Successful Concurrent ISA query", [self._dss_wrapper.participant_id] + ) as main_check: + for isa_id, fetched_isa in results: + if fetched_isa.status_code != 200: + main_check.record_failed( + f"ISA retrieval query failed for {isa_id}", + severity=Severity.High, + details=f"ISA retrieval query for {isa_id} yielded code {fetched_isa.status_code}", + ) + + isa_validator = ISAValidator( + main_check=main_check, + scenario=self, + isa_params=self._isa_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + ) + + for isa_id, fetched_isa in results: + isa_validator.validate_fetched_isa( + isa_id, fetched_isa, expected_version=self._isa_versions[isa_id] + ) + + def _wrap_isa_get_query(self, q: Query) -> FetchedISA: + """Wrap things into the correct utility class""" + if self._dss.rid_version == RIDVersion.f3411_19: + return FetchedISA(v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + return FetchedISA(v22a_query=q) + else: + raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + + def _wrap_isa_put_query(self, q: Query, mutation: str) -> ChangedISA: + """Wrap things into the correct utility class""" + if self._dss.rid_version == RIDVersion.f3411_19: + return ChangedISA(mutation=mutation, v19_query=q) + elif self._dss.rid_version == RIDVersion.f3411_22a: + return ChangedISA(mutation=mutation, v22a_query=q) + else: + raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + + async def _get_isa(self, isa_id): + async with SEMAPHORE: + (_, url) = mutate.build_isa_url(self._dss.rid_version, isa_id) + # Build a `Request` object to register the query later on, + # although we don't need it to do the effective request here on the async_session + # This one is quite barebone and we need to check if anything needs to be added + r = requests.Request( + "GET", + url, + ) + prep = self._dss.client.prepare_request(r) + t0 = datetime.utcnow() + req_descr = describe_request(prep, t0) + status, headers, resp_json = await self._async_session.get_with_headers( + url=url, scope=self._read_scope() + ) + duration = datetime.utcnow() - t0 + rq = Query( + request=req_descr, + response=describe_aiohttp_response( + status, headers, resp_json, duration + ), + participant_id=self._dss.participant_id, + ) + return isa_id, self._wrap_isa_get_query(rq) + + async def _create_isa(self, isa_id): + async with SEMAPHORE: + payload = mutate.build_isa_payload( + **self._isa_params, + rid_version=self._dss.rid_version, + ) + (_, url) = mutate.build_isa_url(self._dss.rid_version, isa_id) + r = requests.Request( + "PUT", + url, + json=payload, + ) + prep = self._dss.client.prepare_request(r) + t0 = datetime.utcnow() + req_descr = describe_request(prep, t0) + status, headers, resp_json = await self._async_session.put_with_headers( + url=url, json=payload, scope=self._write_scope() + ) + duration = datetime.utcnow() - t0 + rq = Query( + request=req_descr, + response=describe_aiohttp_response( + status, headers, resp_json, duration + ), + participant_id=self._dss.participant_id, + ) + return isa_id, self._wrap_isa_put_query(rq, "create") + + async def _delete_isa(self, isa_id, isa_version): + async with SEMAPHORE: + (_, url) = mutate.build_isa_url(self._dss.rid_version, isa_id, isa_version) + r = requests.Request( + "DELETE", + url, + ) + prep = self._dss.client.prepare_request(r) + t0 = datetime.utcnow() + req_descr = describe_request(prep, t0) + status, headers, resp_json = await self._async_session.delete_with_headers( + url=url, scope=self._write_scope() + ) + duration = datetime.utcnow() - t0 + rq = Query( + request=req_descr, + response=describe_aiohttp_response( + status, headers, resp_json, duration + ), + participant_id=self._dss.participant_id, + ) + return isa_id, self._wrap_isa_put_query(rq, "delete") + + def _write_scope(self): + if self._dss.rid_version == RIDVersion.f3411_19: + return v19.constants.Scope.Write + elif self._dss.rid_version == RIDVersion.f3411_22a: + return v22a.constants.Scope.ServiceProvider + else: + raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + + def _read_scope(self): + if self._dss.rid_version == RIDVersion.f3411_19: + return v19.constants.Scope.Read + elif self._dss.rid_version == RIDVersion.f3411_22a: + return v22a.constants.Scope.DisplayProvider + else: + raise ValueError(f"Unsupported RID version '{self._dss.rid_version}'") + + def _create_isas_concurrent_step(self): + loop = asyncio.get_event_loop() + results = loop.run_until_complete( + asyncio.gather(*[self._create_isa(isa_id) for isa_id in self._isa_ids]) + ) + + results = typing.cast(Dict[str, ChangedISA], results) + + for _, fetched_isa in results: + self.record_query(fetched_isa.query) + + with self.check( + "Concurrent ISAs creation", [self._dss_wrapper.participant_id] + ) as main_check: + for isa_id, changed_isa in results: + if changed_isa.query.response.code != 200: + main_check.record_failed( + f"ISA creation failed for {isa_id}", + severity=Severity.High, + details=f"ISA creation for {isa_id} returned {changed_isa.query.response.code}", + ) + else: + self._isa_versions[isa_id] = changed_isa.isa.version + + isa_validator = ISAValidator( + main_check=main_check, + scenario=self, + isa_params=self._isa_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + ) + + for isa_id, changed_isa in results: + isa_validator.validate_mutated_isa( + isa_id, changed_isa, previous_version=None + ) + + def _search_area_step(self): + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as main_check: + isas = self._dss_wrapper.search_isas( + main_check, + area=self._isa_area, + ) + + with self.check( + "Correct ISAs returned by search", [self._dss_wrapper.participant_id] + ) as sub_check: + for isa_id in self._isa_ids: + if isa_id not in isas.isas.keys(): + sub_check.record_failed( + f"ISAs search did not return ISA {isa_id} that was just created", + severity=Severity.High, + details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()} and is missing some of the created ISAs", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + isa_validator = ISAValidator( + main_check=main_check, + scenario=self, + isa_params=self._isa_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + ) + + isa_validator.validate_searched_isas( + isas, expected_versions=self._isa_versions + ) + + def _delete_isas(self): + loop = asyncio.get_event_loop() + results = loop.run_until_complete( + asyncio.gather( + *[ + self._delete_isa(isa_id, self._isa_versions[isa_id]) + for isa_id in self._isa_ids + ] + ) + ) + + results = typing.cast(Dict[str, ChangedISA], results) + + for _, fetched_isa in results: + self.record_query(fetched_isa.query) + + with self.check( + "ISAs deletion query success", [self._dss_wrapper.participant_id] + ) as main_check: + for isa_id, deleted_isa in results: + if deleted_isa.query.response.code != 200: + main_check.record_failed( + f"ISA deletion failed for {isa_id}", + severity=Severity.High, + details=f"ISA deletion for {isa_id} returned {deleted_isa.query.response.code}", + ) + + isa_validator = ISAValidator( + main_check=main_check, + scenario=self, + isa_params=self._isa_params, + dss_id=self._dss.participant_id, + rid_version=self._dss.rid_version, + ) + + for isa_id, changed_isa in results: + isa_validator.validate_deleted_isa( + isa_id, changed_isa, expected_version=self._isa_versions[isa_id] + ) + + def _get_deleted_isas(self): + + loop = asyncio.get_event_loop() + results = loop.run_until_complete( + asyncio.gather(*[self._get_isa(isa_id) for isa_id in self._isa_ids]) + ) + + results = typing.cast(Dict[str, ChangedISA], results) + + for _, fetched_isa in results: + self.record_query(fetched_isa.query) + + with self.check("ISAs not found", [self._dss_wrapper.participant_id]) as check: + for isa_id, fetched_isa in results: + if fetched_isa.status_code != 404: + check.record_failed( + f"ISA retrieval succeeded for {isa_id}", + severity=Severity.High, + details=f"ISA retrieval for {isa_id} returned {fetched_isa.status_code}", + query_timestamps=[fetched_isa.query.request.timestamp], + ) + + def _search_deleted_isas(self): + with self.check( + "Successful ISAs search", [self._dss_wrapper.participant_id] + ) as check: + isas = self._dss_wrapper.search_isas( + check, + area=self._isa_area, + ) + + with self.check( + "ISAs not returned by search", [self._dss_wrapper.participant_id] + ) as check: + for isa_id in self._isa_ids: + if isa_id in isas.isas.keys(): + check.record_failed( + f"ISAs search returned deleted ISA {isa_id}", + severity=Severity.High, + details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()} that contained some of the ISAs we had previously deleted.", + query_timestamps=[isas.dss_query.query.request.timestamp], + ) + + def cleanup(self): + self.begin_cleanup() + + self._delete_isas_if_exists() + self._async_session.close() + + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py index c33d76e22d..17e12fc569 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py @@ -1,13 +1,20 @@ -from typing import Optional, List +from datetime import datetime +from typing import Optional, Dict +from monitoring.monitorlib import schema_validation from monitoring.monitorlib.fetch import rid as fetch -from monitoring.monitorlib.geo import LatLngPoint -from monitoring.monitorlib.mutate import rid as mutate +from monitoring.monitorlib.fetch.rid import ISA, FetchedISA, FetchedISAs from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.monitorlib.mutate import rid as mutate +from monitoring.monitorlib.mutate.rid import ChangedISA from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.common_data_definitions import Severity -from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper -from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario +from monitoring.uss_qualifier.scenarios.scenario import ( + GenericTestScenario, + PendingCheck, +) + +MAX_SKEW = 1e-6 # seconds maximum difference between expected and actual timestamps def delete_isa_if_exists( @@ -67,3 +74,248 @@ def delete_isa_if_exists( f"Attempting to notify subscriber for ISA {isa_id} at {subscriber_url} resulted in {notification.status_code}", query_timestamps=[notification.query.request.timestamp], ) + + +class ISAValidator(object): + """Wraps the validation logic for an ISA that was returned by the DSS. + It will compare the returned ISA with the parameters specified at its creation. + + Inspired by the existing code in DSSWrapper.put_isa() – TODO consider refactoring to merge with it + """ + + _main_check: PendingCheck + _scenario: GenericTestScenario + _isa_params: Dict[str, any] + _dss_id: [str] + _rid_version: RIDVersion + + def __init__( + self, + main_check: PendingCheck, + scenario: GenericTestScenario, + isa_params: Dict[str, any], + dss_id: str, + rid_version: RIDVersion, + ): + self._main_check = main_check + self._scenario = scenario + self._isa_params = isa_params + self._dss_id = [dss_id] + self._rid_version = rid_version + + def _fail_sub_check( + self, _sub_check: PendingCheck, _summary: str, _details: str, t_dss: datetime + ) -> None: + """Fails with Medium severity the sub_check and with High severity the main check.""" + + _sub_check.record_failed( + summary=_summary, + severity=Severity.Medium, + details=_details, + query_timestamps=[t_dss], + ) + + self._main_check.record_failed( + summary=f"ISA request succeeded, but the DSS response is not valid: {_summary}", + severity=Severity.High, + details=_details, + query_timestamps=[t_dss], + ) + + def _validate_isa( + self, + expected_isa_id: str, + dss_isa: ISA, + t_dss: datetime, + previous_version: Optional[ + str + ] = None, # If set, we control that the version changed + expected_version: Optional[ + str + ] = None, # If set, we control that the version has not changed + ) -> None: + isa_id = expected_isa_id + dss_id = self._dss_id + with self._scenario.check("ISA ID matches", dss_id) as sub_check: + if isa_id != dss_isa.id: + self._fail_sub_check( + sub_check, + "DSS did not return correct ISA", + f"Expected ISA ID {dss_id} but got {dss_isa.id}", + t_dss, + ) + + if previous_version is not None: + with self._scenario.check("ISA version changed", dss_id) as sub_check: + if dss_isa.version == previous_version: + self._fail_sub_check( + sub_check, + "ISA version was not updated", + f"Got old version {previous_version} while expecting new version", + t_dss, + ) + + if expected_version is not None: + with self._scenario.check("ISA version matches", dss_id) as sub_check: + if dss_isa.version != expected_version: + self._fail_sub_check( + sub_check, + "ISA version is not the previously held one, although no modification was done to the ISA", + f"Got old version {dss_isa.version} while expecting {expected_version}", + t_dss, + ) + + with self._scenario.check("ISA version format", dss_id) as sub_check: + if not all(c not in "\0\t\r\n#%/:?@[\]" for c in dss_isa.version): + self._fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with invalid version format", + f"DSS returned an ISA with a version that is not URL-safe: {dss_isa.version}", + t_dss, + ) + + with self._scenario.check("ISA start time matches", dss_id) as sub_check: + expected_start = self._isa_params["start_time"] + if abs((dss_isa.time_start - expected_start).total_seconds()) > MAX_SKEW: + self._fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with incorrect start time", + f"DSS should have returned an ISA with a start time of {expected_start}, but instead the ISA returned had a start time of {dss_isa.time_start}", + t_dss, + ) + + with self._scenario.check("ISA end time matches", dss_id) as sub_check: + expected_end = self._isa_params["end_time"] + if abs((dss_isa.time_end - expected_end).total_seconds()) > MAX_SKEW: + self._fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with incorrect end time", + f"DSS should have returned an ISA with an end time of {expected_end}, but instead the ISA returned had an end time of {dss_isa.time_end}", + t_dss, + ) + + with self._scenario.check("ISA URL matches", dss_id) as sub_check: + expected_flights_url = self._rid_version.flights_url_of( + self._isa_params["uss_base_url"] + ) + actual_flights_url = dss_isa.flights_url + if actual_flights_url != expected_flights_url: + self._fail_sub_check( + sub_check, + f"DSS returned ISA (ID {isa_id}) with incorrect URL", + f"DSS should have returned an ISA with a flights URL of {expected_flights_url}, but instead the ISA returned had a flights URL of {actual_flights_url}", + t_dss, + ) + + def validate_fetched_isa( + self, + expected_isa_id: str, + fetched_isa: FetchedISA, + expected_version: str, + ): + """Validates the DSS reply to an ISA fetch request.""" + t_dss = fetched_isa.query.request.timestamp + + with self._scenario.check("ISA response format", self._dss_id) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_get_isa_response_path, + fetched_isa.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + self._fail_sub_check( + sub_check, + "GET ISA response format was invalid", + "Found the following schema validation errors in the DSS response:\n" + + details, + t_dss, + ) + + self._validate_isa( + expected_isa_id, fetched_isa.isa, t_dss, expected_version=expected_version + ) + + def validate_mutated_isa( + self, + expected_isa_id: str, + mutated_isa: ChangedISA, + previous_version: Optional[str] = None, + ): + """ + Validates the DSS reply to an ISA mutation request. + Note that both creating or updating an ISA count as a mutation: the only difference from the + perspective of this function is that previous_version is set in the case of a mutation and None + in the case of a creation. + """ + t_dss = mutated_isa.query.request.timestamp + + with self._scenario.check("ISA response format", self._dss_id) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_put_isa_response_path, + mutated_isa.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + sub_check.record_failed( + "PUT ISA response format was invalid", + Severity.Medium, + "Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) + + self._validate_isa( + expected_isa_id, + mutated_isa.isa, + t_dss, + previous_version=previous_version, + expected_version=None, + ) + + def validate_deleted_isa( + self, + expected_isa_id: str, + deleted_isa: ChangedISA, + expected_version: str, + ): + """Validates the DSS reply to an ISA deletion request.""" + t_dss = deleted_isa.query.request.timestamp + + with self._scenario.check("ISA response format", self._dss_id) as sub_check: + errors = schema_validation.validate( + self._rid_version.openapi_path, + self._rid_version.openapi_delete_isa_response_path, + deleted_isa.query.response.json, + ) + if errors: + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + sub_check.record_failed( + "PUT ISA response format was invalid", + Severity.Medium, + "Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + ) + + self._validate_isa( + expected_isa_id, deleted_isa.isa, t_dss, expected_version=expected_version + ) + + def validate_searched_isas( + self, + fetched_isas: FetchedISAs, + expected_versions: Dict[str, str], + ): + """Validates the DSS reply to an ISA search request: + based on the ISA ID's present in expected_versions, it will verify the content of the returned ISA's. + Note that ISAs that are not part of the test are entirely ignored. + """ + for isa_id, isa_version in expected_versions.items(): + self._validate_isa( + isa_id, + fetched_isas.isas[isa_id], + fetched_isas.query.request.timestamp, + expected_version=expected_versions[isa_id], + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py index aed0f85b9d..a89cb054cb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py @@ -6,3 +6,4 @@ from .subscription_simple import SubscriptionSimple from .token_validation import TokenValidation from .crdb_access import CRDBAccess +from .heavy_traffic_concurrent import HeavyTrafficConcurrent diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md new file mode 100644 index 0000000000..7757a0302e --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md @@ -0,0 +1,188 @@ +# ASTM NetRID DSS: Concurrent Requests test scenario + +## Overview + +Create, query and delete ISAs on the DSS, concurrently. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing the ISAs to be created. All created ISAs use the same parameters. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates ISA's with known IDs. This step ensures that no ISA with a known ID is present in the DSS before proceeding with the test. + +#### Successful ISA query check + +While F3411-19 does not explicitly require the implementation of a specific ISA retrieval endpoint, Annex A4 specifies the explicit format for this endpoint. If this format is not followed and the error isn't a 404, this check will fail per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. + +## Concurrent Requests test case + +This test case will: + +1. Create ISAs concurrently +2. Query each ISA individually, but concurrently +3. Search for all ISAs in the area of the created ISAs (using a single request) +4. Delete the ISAs concurrently +5. Query each ISA individually, but concurrently +6. Search for all ISAs in the area of the deleted ISAs (using a single request) + +### [Create ISA concurrently test step](test_steps/put_isa.md) + +This step attempts to concurrently create multiple ISAs, as specified in this scenario's resource, at the configured DSS. + +#### Concurrent ISAs creation check + +If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +### Get ISAs concurrently test step + +This step attempts to concurrently retrieve the previously created ISAs from the DSS. + +#### Successful Concurrent ISA query check + +If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. + +#### ISA response format check + +The API for **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### ISA ID matches check + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +#### ISA version matches check + +The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +### [Search Available ISAs test step](test_steps/search_isas.md) + +This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### Correct ISAs returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. + +#### ISA ID matches check + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA version matches check + +The DSS returns the version of the ISA in the response body. If this version has changed without an update of the ISA, **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +#### ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +### [Delete ISAs concurrently test step](test_steps/delete_isa.md) + +This step attempts to concurrently delete the earlier created ISAs. + +#### ISAs deletion query success check + +If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +#### ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +#### ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. + +### Access Deleted ISAs test step + +This step attempts to concurrently access the previously deleted ISAs from the DSS. + +#### ISAs not found check + +The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +### [Search Deleted ISAs test step](test_steps/search_isas.md) + +This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### ISAs not returned by search check + +The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove any created ISA if the test ended prematurely. + +### Successful ISA query check + +**[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +### Removed pre-existing ISA check + +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +### Notified subscriber check + +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py new file mode 100644 index 0000000000..9eee0c4b76 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.heavy_traffic_concurrent import ( + HeavyTrafficConcurrent as CommonHeavyTrafficConcurrent, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class HeavyTrafficConcurrent(TestScenario, CommonHeavyTrafficConcurrent): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py index aed0f85b9d..a89cb054cb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py @@ -6,3 +6,4 @@ from .subscription_simple import SubscriptionSimple from .token_validation import TokenValidation from .crdb_access import CRDBAccess +from .heavy_traffic_concurrent import HeavyTrafficConcurrent diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md new file mode 100644 index 0000000000..3c7177ea0a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md @@ -0,0 +1,188 @@ +# ASTM NetRID DSS: Concurrent Requests test scenario + +## Overview + +Create, query and delete ISAs on the DSS, concurrently. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing the ISAs to be created. All created ISAs use the same parameters. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates ISA's with known IDs. This step ensures that no ISA with a known ID is present in the DSS before proceeding with the test. + +#### Successful ISA query check + +While F3411-19 does not explicitly require the implementation of a specific ISA retrieval endpoint, Annex A4 specifies the explicit format for this endpoint. If this format is not followed and the error isn't a 404, this check will fail per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. + +## Concurrent Requests test case + +This test case will: + +1. Create ISAs concurrently +2. Query each ISA individually, but concurrently +3. Search for all ISAs in the area of the created ISAs (using a single request) +4. Delete the ISAs concurrently +5. Query each ISA individually, but concurrently +6. Search for all ISAs in the area of the deleted ISAs (using a single request) + +### [Create ISA concurrently test step](test_steps/put_isa.md) + +This step attempts to concurrently create multiple ISAs, as specified in this scenario's resource, at the configured DSS. + +#### Concurrent ISAs creation check + +If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +### Get ISAs concurrently test step + +This step attempts to concurrently retrieve the previously created ISAs from the DSS. + +#### Successful Concurrent ISA query check + +If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. + +#### ISA response format check + +The API for **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** specifies an explicit format that the DSS responses must follow. If the DSS response does not validate against this format, this check will fail. + +#### ISA ID matches check + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +#### ISA version matches check + +The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +### [Search Available ISAs test step](test_steps/search_isas.md) + +This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### Correct ISAs returned by search check + +The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. + +#### ISA ID matches check + +The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA version matches check + +The DSS returns the version of the ISA in the response body. If this version has changed without an update of the ISA, **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. + +#### ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +#### ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +### [Delete ISAs concurrently test step](test_steps/delete_isa.md) + +This step attempts to concurrently delete the earlier created ISAs. + +#### ISAs deletion query success check + +If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### ISA version format check + +Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. + +#### ISA start time matches check + +The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### ISA end time matches check + +The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +#### ISA URL matches check + +When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. + +### Access Deleted ISAs test step + +This step attempts to concurrently access the previously deleted ISAs from the DSS. + +#### ISAs not found check + +The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +### [Search Deleted ISAs test step](test_steps/search_isas.md) + +This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. + +#### Successful ISAs search check + +The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +#### ISAs not returned by search check + +The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove any created ISA if the test ended prematurely. + +### Successful ISA query check + +**[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +### Removed pre-existing ISA check + +If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +### Notified subscriber check + +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.py new file mode 100644 index 0000000000..9eee0c4b76 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.heavy_traffic_concurrent import ( + HeavyTrafficConcurrent as CommonHeavyTrafficConcurrent, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class HeavyTrafficConcurrent(TestScenario, CommonHeavyTrafficConcurrent): + pass diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 7c6f4a358d..185f6ae133 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -24,12 +24,12 @@ astm
.f3411
.v19 DSS0030,a Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation DSS0030,b Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation DSS0030,c @@ -329,7 +329,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation interuss
.automated_testing
.rid
.injection @@ -367,11 +367,11 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index 601f9c6b09..485b008ad1 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -13,6 +13,7 @@ 7. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 8. Scenario: [ASTM NetRID DSS: Token Validation](../../../../scenarios/astm/netrid/v19/dss/token_validation.md) ([`scenarios.astm.netrid.v19.dss.TokenValidation`](../../../../scenarios/astm/netrid/v19/dss/token_validation.py)) 9. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) +10. Scenario: [ASTM NetRID DSS: Concurrent Requests](../../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md) ([`scenarios.astm.netrid.v19.dss.HeavyTrafficConcurrent`](../../../../scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.py)) ## [Checked requirements](../../../README.md#checked-requirements) @@ -27,12 +28,12 @@ astm
.f3411
.v19 DSS0030,a Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation DSS0030,b Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation DSS0030,c @@ -207,17 +208,17 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml index 3edcffd08d..78918859b6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml @@ -60,3 +60,9 @@ actions: scenario_type: scenarios.astm.netrid.v19.dss.CRDBAccess resources: {} on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.dss.HeavyTrafficConcurrent + resources: + dss: dss + id_generator: id_generator + isa: isa diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index e62aca6669..74282404f8 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -45,12 +45,12 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c @@ -470,7 +470,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.automated_testing
.rid
.injection @@ -508,11 +508,11 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md index 3a67d430fd..a8a2f25bdd 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md @@ -13,6 +13,7 @@ 7. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 8. Scenario: [ASTM NetRID DSS: Token Validation](../../../../scenarios/astm/netrid/v19/dss/token_validation.md) ([`scenarios.astm.netrid.v19.dss.TokenValidation`](../../../../scenarios/astm/netrid/v19/dss/token_validation.py)) 9. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v22a/dss/crdb_access.md) ([`scenarios.astm.netrid.v22a.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v22a/dss/crdb_access.py)) +10. Scenario: [ASTM NetRID DSS: Concurrent Requests](../../../../scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md) ([`scenarios.astm.netrid.v22a.dss.HeavyTrafficConcurrent`](../../../../scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.py)) ## [Checked requirements](../../../README.md#checked-requirements) @@ -48,12 +49,12 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c @@ -233,17 +234,17 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml index fe94dc34c6..75b5040334 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml @@ -60,3 +60,9 @@ actions: scenario_type: scenarios.astm.netrid.v22a.dss.CRDBAccess resources: {} on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v22a.dss.HeavyTrafficConcurrent + resources: + dss: dss + id_generator: id_generator + isa: isa diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md index 3d5ca942be..1cb4e858e6 100644 --- a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md +++ b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md @@ -22,12 +22,12 @@ astm
.f3411
.v19 DSS0030,a Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation DSS0030,b Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation DSS0030,c @@ -202,7 +202,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation astm
.f3411
.v22a @@ -213,12 +213,12 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c @@ -398,17 +398,17 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index f6862800f1..4c6abe3b89 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -40,12 +40,12 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c @@ -465,7 +465,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.automated_testing
.rid
.injection @@ -503,11 +503,11 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index a46e8fe057..710953fe93 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -42,12 +42,12 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c @@ -467,7 +467,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations astm
.f3548
.v21 @@ -642,12 +642,12 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations
ASTM NetRID DSS: Token Validation SearchISAs Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior + ASTM NetRID DSS: Concurrent Requests
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior interuss
.f3548
.notification_requirements diff --git a/schemas/monitoring/uss_qualifier/resources/interuss/mock_uss/client/MockUSSSpecification.json b/schemas/monitoring/uss_qualifier/resources/interuss/mock_uss/client/MockUSSSpecification.json index 30f7b4cc0c..c50bf11445 100644 --- a/schemas/monitoring/uss_qualifier/resources/interuss/mock_uss/client/MockUSSSpecification.json +++ b/schemas/monitoring/uss_qualifier/resources/interuss/mock_uss/client/MockUSSSpecification.json @@ -14,6 +14,13 @@ "participant_id": { "description": "Test participant responsible for this mock USS.", "type": "string" + }, + "timeout_seconds": { + "description": "Number of seconds to allow for requests to this mock_uss instance. If None, use default.", + "type": [ + "number", + "null" + ] } }, "required": [