diff --git a/monitoring/monitorlib/infrastructure.py b/monitoring/monitorlib/infrastructure.py index 1e789d3263..5c8f1de086 100644 --- a/monitoring/monitorlib/infrastructure.py +++ b/monitoring/monitorlib/infrastructure.py @@ -133,6 +133,9 @@ def request(self, method, url, **kwargs): return super().request(method, url, **kwargs) + def get_prefix_url(self): + return self._prefix_url + class AsyncUTMTestSession: """ diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py new file mode 100644 index 0000000000..fe75b44ebf --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -0,0 +1,342 @@ +import time +import traceback +import uuid +from typing import List + +import arrow +import s2sphere +from implicitdict import ImplicitDict +from loguru import logger +from requests.exceptions import RequestException +from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse + +from monitoring.monitorlib import fetch +from monitoring.monitorlib.fetch import rid +from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.monitorlib.rid import RIDVersion +from monitoring.monitorlib.rid_automated_testing.injection_api import ( + CreateTestParameters, +) +from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource +from monitoring.uss_qualifier.resources.netrid import ( + FlightDataResource, + NetRIDServiceProviders, + EvaluationConfigurationResource, +) +from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator +from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( + InjectedFlightCollection, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.injection import ( + InjectedFlight, + InjectedTest, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( + VirtualObserver, +) +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + + +class Misbehavior(GenericTestScenario): + """ + Check that an unauthenticated client is not able to query a Service Provider + """ + + _flights_data: FlightDataResource + _service_providers: NetRIDServiceProviders + _evaluation_configuration: EvaluationConfigurationResource + _injected_flights: List[InjectedFlight] + _injected_tests: List[InjectedTest] + + def __init__( + self, + flights_data: FlightDataResource, + service_providers: NetRIDServiceProviders, + evaluation_configuration: EvaluationConfigurationResource, + dss_pool: DSSInstancesResource = None, + ): + super().__init__() + self._flights_data = flights_data + self._service_providers = service_providers + self._evaluation_configuration = evaluation_configuration + self._injected_flights = [] + self._injected_tests = [] + if len(dss_pool.dss_instances) == 0: + raise ValueError( + "The Misbehavior Scenario requires at least one DSS instance" + ) + self._dss = dss_pool.dss_instances[0] + + @property + def _rid_version(self) -> RIDVersion: + raise NotImplementedError( + "Misbehavior test scenario subclass must specify _rid_version" + ) + + def run(self): + self.begin_test_scenario() + self.begin_test_case("Unauthenticated requests") + + self.begin_test_step("Injection") + self._inject_flights() + self.end_test_step() + + self.begin_test_step("Unauthenticated requests") + + self._poll_unauthenticated_during_flights() + + self.end_test_step() + + self.end_test_case() + self.end_test_scenario() + + def _inject_flights(self): + # TODO consider sharing code with nominal_behavior's _inject_flights + # Inject flights into all USSs + test_id = str(uuid.uuid4()) + test_flights = self._flights_data.get_test_flights() + service_providers = self._service_providers.service_providers + if len(service_providers) > len(test_flights): + raise ValueError( + "{} service providers were specified, but data for only {} test flights were provided".format( + len(service_providers), len(test_flights) + ) + ) + for i, target in enumerate(service_providers): + p = CreateTestParameters(requested_flights=[test_flights[i]]) + check = self.check("Successful injection", [target.participant_id]) + try: + query = target.submit_test(p, test_id) + except RequestException as e: + stacktrace = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + check.record_failed( + summary="Error while trying to inject test flight", + severity=Severity.High, + details=f"While trying to inject a test flight into {target.participant_id}, encountered error:\n{stacktrace}", + ) + raise RuntimeError("High-severity issue did not abort test scenario") + self.record_query(query) + try: + if query.status_code != 200: + raise ValueError( + f"Expected response code 200 but received {query.status_code} instead" + ) + if "json" not in query.response: + raise ValueError("Response did not contain a JSON body") + changed_test: ChangeTestResponse = ImplicitDict.parse( + query.response.json, ChangeTestResponse + ) + self._injected_tests.append( + InjectedTest( + participant_id=target.participant_id, + test_id=test_id, + version=changed_test.version, + ) + ) + injections = changed_test.injected_flights + check.record_passed() + except ValueError as e: + check.record_failed( + summary="Error injecting test flight", + severity=Severity.High, + details=f"Attempting to inject a test flight into {target.participant_id}, encountered status code {query.status_code}: {str(e)}", + query_timestamps=[query.request.timestamp], + ) + raise RuntimeError("High-severity issue did not abort test scenario") + + for flight in injections: + self._injected_flights.append( + InjectedFlight( + uss_participant_id=target.participant_id, + test_id=test_id, + flight=TestFlight(flight), + query_timestamp=query.request.timestamp, + ) + ) + + # Make sure the injected flights can be identified correctly by the test harness + with self.check("Identifiable flights") as check: + errors = display_data_evaluator.injected_flights_errors( + self._injected_flights + ) + if errors: + check.record_failed( + "Injected flights not suitable for test", + Severity.High, + details="When checking the suitability of the flights (as injected) for the test, found:\n" + + "\n".join(errors), + query_timestamps=[ + f.query_timestamp for f in self._injected_flights + ], + ) + raise RuntimeError("High-severity issue did not abort test scenario") + + config = self._evaluation_configuration.configuration + self._virtual_observer = VirtualObserver( + injected_flights=InjectedFlightCollection(self._injected_flights), + repeat_query_rect_period=config.repeat_query_rect_period, + min_query_diagonal_m=config.min_query_diagonal, + relevant_past_data_period=self._rid_version.realtime_period + + config.max_propagation_latency.timedelta, + ) + + def _poll_unauthenticated_during_flights(self): + config = self._evaluation_configuration.configuration + + t_end = self._virtual_observer.get_last_time_of_interest() + t_now = arrow.utcnow() + + if t_now > t_end: + raise RuntimeError( + f"Cannot evaluate RID system: injected test flights ended at {t_end}, which is before now ({t_now})" + ) + + logger.debug(f"Polling from {t_now} until {t_end}") + for f in self._injected_flights: + span = f.flight.get_span() + logger.debug( + f"Flight {f.uss_participant_id}/{f.flight.injection_id} {span[0].isoformat()} to {span[1].isoformat()}", + ) + + t_next = arrow.utcnow() + dt = config.min_polling_interval.timedelta + while arrow.utcnow() < t_end: + # Evaluate the system at an instant in time for various areas + diagonals_m = [ + self._rid_version.max_diagonal_km * 1000 + 500, # too large + self._rid_version.max_diagonal_km * 1000 - 100, # clustered + self._rid_version.max_details_diagonal_km * 1000 - 100, # details + ] + auth_tests = [] + for diagonal_m in diagonals_m: + rect = self._virtual_observer.get_query_rect(diagonal_m) + auth_tests.append(self._evaluate_and_test_authentication(rect)) + + # If we checked for all diagonals that flights queries are properly authenticated, + # we can stop polling + if all(auth_tests): + logger.debug( + "Authentication check is complete, ending polling now.", + ) + break + + # Wait until minimum polling interval elapses + while t_next < arrow.utcnow(): + t_next += dt + if t_next > t_end: + break + delay = t_next - arrow.utcnow() + if delay.total_seconds() > 0: + logger.debug( + f"Waiting {delay.total_seconds()} seconds before polling RID system again..." + ) + time.sleep(delay.total_seconds()) + + def _evaluate_and_test_authentication( + self, + rect: s2sphere.LatLngRect, + ) -> bool: + """Queries all flights in the expected way, then repeats the queries to SPs without credentials. + + returns true once queries to SPS have been made without credentials. False otherwise, such as when + no flights were yet returned by the authenticated queries. + """ + + with self.check("Missing credentials") as check: + # We grab all flights from the SP's. This is authenticated + # and is expected to succeed + sp_observation = rid.all_flights( + rect, + include_recent_positions=True, + get_details=True, + rid_version=self._rid_version, + session=self._dss.client, + ) + # We fish out the queries that were used to grab the flights from the SP, + # and attempt to re-query without credentials. This should fail. + + unauthenticated_session = UTMClientSession( + prefix_url=self._dss.client.get_prefix_url(), + auth_adapter=None, + timeout_seconds=self._dss.client.timeout_seconds, + ) + + queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list( + sp_observation.uss_flight_details_queries.values() + ) + + if len(queries_to_repeat) == 0: + logger.debug("no flights queries to repeat at this point.") + return False + + logger.debug( + f"about to repeat {len(queries_to_repeat)} flights queries without credentials" + ) + + # Attempt to re-query the flights and flight details URLs: + for fq in queries_to_repeat: + failed_q = fetch.query_and_describe( + client=unauthenticated_session, + verb=fq.query.request.method, + url=fq.query.request.url, + json=fq.query.request.json, + data=fq.query.request.body, + ) + logger.info( + f"Repeating query to {fq.query.request.url} without credentials" + ) + server_id = fq.query.get("server_id", "unknown") + if failed_q.response.code not in [401, 403]: + check.record_failed( + "unauthenticated request was fulfilled", + participants=[server_id], + severity=Severity.MEDIUM, + details=f"queried flights on {fq.query.request.url} with no credentials, expected a failure but got a success reply", + ) + else: + logger.info( + f"participant with id {server_id} properly authenticated the request" + ) + # Keep track of the failed queries, too + self.record_query(failed_q) + + return True + + def cleanup(self): + self.begin_cleanup() + while self._injected_tests: + injected_test = self._injected_tests.pop() + matching_sps = [ + sp + for sp in self._service_providers.service_providers + if sp.participant_id == injected_test.participant_id + ] + if len(matching_sps) != 1: + matching_ids = ", ".join(sp.participant_id for sp in matching_sps) + raise RuntimeError( + f"Found {len(matching_sps)} service providers with participant ID {injected_test.participant_id} ({matching_ids}) when exactly 1 was expected" + ) + sp = matching_sps[0] + check = self.check("Successful test deletion", [sp.participant_id]) + try: + query = sp.delete_test(injected_test.test_id, injected_test.version) + self.record_query(query) + if query.status_code != 200: + raise ValueError( + f"Received status code {query.status_code} after attempting to delete test {injected_test.test_id} at version {injected_test.version} from service provider {sp.participant_id}" + ) + check.record_passed() + except (RequestException, ValueError) as e: + stacktrace = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + check.record_failed( + summary="Error while trying to delete test flight", + severity=Severity.Medium, + details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}", + ) + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index 92be3e719f..090a654bed 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -1,20 +1,19 @@ -from datetime import timedelta +import time import traceback import uuid -import time from typing import List, Optional import arrow from implicitdict import ImplicitDict from loguru import logger from requests.exceptions import RequestException -from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight -from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario +from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse +from monitoring.monitorlib.rid import RIDVersion from monitoring.monitorlib.rid_automated_testing.injection_api import ( CreateTestParameters, ) -from monitoring.monitorlib.rid import RIDVersion +from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource from monitoring.uss_qualifier.resources.netrid import ( @@ -23,19 +22,18 @@ NetRIDObserversResource, EvaluationConfigurationResource, ) +from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( InjectedFlightCollection, ) -from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( - VirtualObserver, -) -from monitoring.uss_qualifier.scenarios.scenario import TestScenario -from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator from monitoring.uss_qualifier.scenarios.astm.netrid.injection import ( InjectedFlight, InjectedTest, ) -from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse +from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( + VirtualObserver, +) +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario class NominalBehavior(GenericTestScenario): @@ -66,10 +64,8 @@ def __init__( @property def _rid_version(self) -> RIDVersion: - return ( - self._dss_pool.dss_instances[0].rid_version - if self._dss_pool - else RIDVersion.f3411_19 + raise NotImplementedError( + "NominalBehavior test scenario subclass must specify _rid_version" ) def run(self): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 67765986bb..82265dbb32 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -36,7 +36,11 @@ from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( VirtualObserver, ) -from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType, PendingCheck +from monitoring.uss_qualifier.scenarios.scenario import ( + TestScenarioType, + PendingCheck, + TestScenario, +) from monitoring.uss_qualifier.scenarios.astm.netrid.injection import InjectedFlight DISTANCE_TOLERANCE_M = 0.01 @@ -190,7 +194,7 @@ class RIDObservationEvaluator(object): def __init__( self, - test_scenario: TestScenarioType, + test_scenario: TestScenario, injected_flights: List[InjectedFlight], config: EvaluationConfiguration, rid_version: RIDVersion, diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py index 1552d4ee7d..ab08466a88 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py @@ -1,3 +1,4 @@ from .dss_interoperability import DSSInteroperability from .nominal_behavior import NominalBehavior +from .misbehavior import Misbehavior from .aggregate_checks import AggregateChecks diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md new file mode 100644 index 0000000000..0602647f31 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -0,0 +1,60 @@ +# ASTM NetRID SP clients misbehavior handling test scenario + +## Overview + +In this scenario, the service provider's endpoint are accessed directly with missing or incorrect credentials. Resources that exists as well as resources that are not expected to exist are queried. + +## Resources + +### flights_data + +A [`FlightDataResource`](../../../../resources/netrid/flight_data.py) containing 1 nominal flight per SP under test. + +### service_providers + +A set of [`NetRIDServiceProviders`](../../../../resources/netrid/service_providers.py) to be tested for proper request authentication. This scenario requires at least one SP under test. + +### evaluation_configuration + +This [`EvaluationConfigurationResource`](../../../../resources/netrid/evaluation.py) defines how to gauge success when observing the injected flights. + +### dss_pool + +A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required for providing the qualifier with the flights URL of the service providers being tested. + +## Unauthenticated requests test case + +### Injection test step + +In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. + +#### Successful injection check + +Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. + +**[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. +This check will fail if the flight was not successfully injected. + +#### Identifiable flights check + +This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. + +### Unauthenticated requests test step + +In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials +to confirm that the requested data is indeed available to any authorized query. + +It then repeats the exact same request while omitting the credentials, and expects this to fail. + +#### Missing credentials check + +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)**, +and that requests for existing flights that are executed with missing or incorrect credentials fail. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove injected data from all SPs. + +### Successful test deletion check + +**[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py new file mode 100644 index 0000000000..a45925f919 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py @@ -0,0 +1,11 @@ +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid.common.misbehavior import ( + Misbehavior as CommonMisbehavior, +) + + +class Misbehavior(TestScenario, CommonMisbehavior): + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_19 diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py index 19713cd1bb..4ce255a732 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py @@ -1,3 +1,4 @@ +from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.scenarios.astm.netrid.common.nominal_behavior import ( NominalBehavior as CommonNominalBehavior, @@ -5,4 +6,6 @@ class NominalBehavior(TestScenario, CommonNominalBehavior): - pass + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_19 diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md new file mode 100644 index 0000000000..d94a021a73 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -0,0 +1,60 @@ +# ASTM NetRID SP clients misbehavior handling test scenario + +## Overview + +In this scenario, the service provider's endpoint are accessed directly with missing or incorrect credentials. Resources that exists as well as resources that are not expected to exist are queried. + +## Resources + +### flights_data + +A [`FlightDataResource`](../../../../resources/netrid/flight_data.py) containing 1 nominal flight per SP under test. + +### service_providers + +A set of [`NetRIDServiceProviders`](../../../../resources/netrid/service_providers.py) to be tested for proper request authentication. This scenario requires at least one SP under test. + +### evaluation_configuration + +This [`EvaluationConfigurationResource`](../../../../resources/netrid/evaluation.py) defines how to gauge success when observing the injected flights. + +### dss_pool + +A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required for providing the qualifier with the flights URL of the service providers being tested. + +## Unauthenticated requests test case + +### Injection test step + +In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. + +#### Successful injection check + +Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. + +**[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. +This check will fail if the flight was not successfully injected. + +#### Identifiable flights check + +This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. + +### Unauthenticated requests test step + +In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials +to confirm that the requested data is indeed available to any authorized query. + +It then repeats the exact same request while omitting the credentials, and expects this to fail. + +#### Missing credentials check + +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)**, +and that requests for existing flights that are executed with missing or incorrect credentials fail. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove injected data from all SPs. + +### Successful test deletion check + +**[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py new file mode 100644 index 0000000000..c687ea7caa --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py @@ -0,0 +1,11 @@ +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid.common.misbehavior import ( + Misbehavior as CommonMisbehavior, +) + + +class Misbehavior(TestScenario, CommonMisbehavior): + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_22a diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py index 19713cd1bb..c0110898e4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py @@ -1,3 +1,4 @@ +from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.scenarios.astm.netrid.common.nominal_behavior import ( NominalBehavior as CommonNominalBehavior, @@ -5,4 +6,6 @@ class NominalBehavior(TestScenario, CommonNominalBehavior): - pass + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_22a diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 7bddb15718..3d6500092d 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -8,7 +8,8 @@ 1. Suite: [DSS instance probing for ASTM NetRID F3411-19](f3411_19/dss_probing.md) ([`suites.astm.netrid.f3411_19.dss_probing`](f3411_19/dss_probing.yaml)) 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) -3. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) +4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) +4. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) ## Checked requirements @@ -223,7 +224,7 @@ NET0500 Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior NET0610 @@ -239,7 +240,7 @@ interuss.automated_testing.rid.injection DeleteTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior ExpectedBehavior @@ -254,7 +255,7 @@ UpsertTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior interuss.automated_testing.rid.observation diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml index 5404323390..70e5e81486 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml @@ -38,6 +38,15 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.Misbehavior + resources: + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + dss_pool: dss_instances + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v19.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 55ada4d22f..c289870712 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -8,7 +8,8 @@ 1. Suite: [DSS instance probing for ASTM NetRID F3411-22a](f3411_22a/dss_probing.md) ([`suites.astm.netrid.f3411_22a.dss_probing`](f3411_22a/dss_probing.yaml)) 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) -3. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) +4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) +4. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) ## Checked requirements @@ -19,6 +20,12 @@ Status Checked in + + astm.f3411.v19 + NET0500 + Implemented + ASTM NetRID SP clients misbehavior handling + astm.f3411.v22a A2-6-1,1a @@ -279,7 +286,7 @@ interuss.automated_testing.rid.injection DeleteTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior ExpectedBehavior @@ -294,7 +301,7 @@ UpsertTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior interuss.automated_testing.rid.observation diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml index 77411d0577..2bf688260b 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml @@ -40,6 +40,15 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.Misbehavior + resources: + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + dss_pool: dss_instances + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v22a.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 7d84eea167..3d1e73ffae 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -15,6 +15,12 @@ Status Checked in + + astm.f3411.v19 + NET0500 + Implemented + ASTM NetRID SP clients misbehavior handling + astm.f3411.v22a A2-6-1,1a @@ -275,7 +281,7 @@ interuss.automated_testing.rid.injection DeleteTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior ExpectedBehavior @@ -290,7 +296,7 @@ UpsertTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior interuss.automated_testing.rid.observation diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index ba7e196d4f..fbaf8dece6 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -16,6 +16,12 @@ Status Checked in + + astm.f3411.v19 + NET0500 + Implemented + ASTM NetRID SP clients misbehavior handling + astm.f3411.v22a A2-6-1,1a @@ -383,7 +389,7 @@ interuss.automated_testing.rid.injection DeleteTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior ExpectedBehavior @@ -398,7 +404,7 @@ UpsertTestSuccess Implemented - ASTM NetRID nominal behavior + ASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior interuss.automated_testing.rid.observation