From 39b0727b989ea9ff7db645815f2211e1b1a9f670 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Wed, 6 Sep 2023 18:32:07 +0200 Subject: [PATCH] NET0210 --- monitoring/monitorlib/infrastructure.py | 3 + .../astm/netrid/common/misbehavior.py | 294 ++++++++++++++++++ .../astm/netrid/display_data_evaluator.py | 75 +++++ .../scenarios/astm/netrid/v19/__init__.py | 1 + .../scenarios/astm/netrid/v19/misbehavior.md | 47 +++ .../scenarios/astm/netrid/v19/misbehavior.py | 8 + .../astm/netrid/v19/nominal_behavior.md | 8 + .../scenarios/astm/netrid/v22a/misbehavior.md | 43 +++ .../scenarios/astm/netrid/v22a/misbehavior.py | 8 + .../suites/astm/netrid/f3411_19.md | 1 + .../suites/astm/netrid/f3411_19.yaml | 9 + .../suites/astm/netrid/f3411_22a.md | 1 + .../suites/astm/netrid/f3411_22a.yaml | 9 + 13 files changed, 507 insertions(+) create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py diff --git a/monitoring/monitorlib/infrastructure.py b/monitoring/monitorlib/infrastructure.py index 1e789d3263..5c8f1de086 100644 --- a/monitoring/monitorlib/infrastructure.py +++ b/monitoring/monitorlib/infrastructure.py @@ -133,6 +133,9 @@ def request(self, method, url, **kwargs): return super().request(method, url, **kwargs) + def get_prefix_url(self): + return self._prefix_url + class AsyncUTMTestSession: """ diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py new file mode 100644 index 0000000000..e243e25b9a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -0,0 +1,294 @@ +from datetime import timedelta +import traceback +import uuid +import time +from typing import List, Optional + +import arrow +from implicitdict import ImplicitDict +from loguru import logger +from requests.exceptions import RequestException + +from monitoring.monitorlib import fetch +from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight +from monitoring.uss_qualifier.resources.netrid.service_area import ( + ServiceAreaSpecification, +) +from monitoring.uss_qualifier.resources.netrid.service_providers import ( + NetRIDServiceProvider, +) +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + +from monitoring.monitorlib.rid_automated_testing.injection_api import ( + CreateTestParameters, +) +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource +from monitoring.uss_qualifier.resources.netrid import ( + FlightDataResource, + NetRIDServiceProviders, + NetRIDObserversResource, + EvaluationConfigurationResource, + ServiceAreaResource, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( + InjectedFlightCollection, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( + VirtualObserver, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator +from monitoring.uss_qualifier.scenarios.astm.netrid.injection import ( + InjectedFlight, + InjectedTest, +) +from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse + + +class Misbehavior(GenericTestScenario): + """ + Check that an unauthenticated client is not able to query a Service Provider + """ + + _flights_data: FlightDataResource + _service_providers: NetRIDServiceProviders + _observers: NetRIDObserversResource + _evaluation_configuration: EvaluationConfigurationResource + + _injected_flights: List[InjectedFlight] + _injected_tests: List[InjectedTest] + + def __init__( + self, + flights_data: FlightDataResource, + service_providers: NetRIDServiceProviders, + observers: NetRIDObserversResource, + evaluation_configuration: EvaluationConfigurationResource, + dss_pool: Optional[DSSInstancesResource] = None, + ): + super().__init__() + self._flights_data = flights_data + self._service_providers = service_providers + self._observers = observers + self._evaluation_configuration = evaluation_configuration + self._injected_flights = [] + self._injected_tests = [] + self._dss_pool = dss_pool + + @property + def _rid_version(self) -> RIDVersion: + return ( + self._dss_pool.dss_instances[0].rid_version + if self._dss_pool + else RIDVersion.f3411_19 + ) + + def run(self): + self.begin_test_scenario() + self.begin_test_case("Unauthenticated requests") + + self.begin_test_step("Injection") + self._inject_flights() + self.end_test_step() + + # TODO should this go into the evaluator? + self.begin_test_step("Unauthenticated requests") + + self._poll_unauthenticated_during_flights() + + self.end_test_step() + + self.end_test_case() + self.end_test_scenario() + + def _inject_flights(self): + # TODO consider sharing code with nominal_behavior's _inject_flights + # Inject flights into all USSs + test_id = str(uuid.uuid4()) + test_flights = self._flights_data.get_test_flights() + service_providers = self._service_providers.service_providers + if len(service_providers) > len(test_flights): + raise ValueError( + "{} service providers were specified, but data for only {} test flights were provided".format( + len(service_providers), len(test_flights) + ) + ) + for i, target in enumerate(service_providers): + p = CreateTestParameters(requested_flights=[test_flights[i]]) + check = self.check("Successful injection", [target.participant_id]) + try: + query = target.submit_test(p, test_id) + except RequestException as e: + stacktrace = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + check.record_failed( + summary="Error while trying to inject test flight", + severity=Severity.High, + details=f"While trying to inject a test flight into {target.participant_id}, encountered error:\n{stacktrace}", + ) + raise RuntimeError("High-severity issue did not abort test scenario") + self.record_query(query) + try: + if query.status_code != 200: + raise ValueError( + f"Expected response code 200 but received {query.status_code} instead" + ) + if "json" not in query.response: + raise ValueError("Response did not contain a JSON body") + changed_test: ChangeTestResponse = ImplicitDict.parse( + query.response.json, ChangeTestResponse + ) + self._injected_tests.append( + InjectedTest( + participant_id=target.participant_id, + test_id=test_id, + version=changed_test.version, + ) + ) + injections = changed_test.injected_flights + check.record_passed() + except ValueError as e: + check.record_failed( + summary="Error injecting test flight", + severity=Severity.High, + details=f"Attempting to inject a test flight into {target.participant_id}, encountered status code {query.status_code}: {str(e)}", + query_timestamps=[query.request.timestamp], + ) + raise RuntimeError("High-severity issue did not abort test scenario") + # TODO: Validate injected flights, especially to make sure they contain the specified injection IDs + for flight in injections: + self._injected_flights.append( + InjectedFlight( + uss_participant_id=target.participant_id, + test_id=test_id, + flight=TestFlight(flight), + query_timestamp=query.request.timestamp, + ) + ) + + # Make sure the injected flights can be identified correctly by the test harness + with self.check("Identifiable flights") as check: + errors = display_data_evaluator.injected_flights_errors( + self._injected_flights + ) + if errors: + check.record_failed( + "Injected flights not suitable for test", + Severity.High, + details="When checking the suitability of the flights (as injected) for the test, found:\n" + + "\n".join(errors), + query_timestamps=[ + f.query_timestamp for f in self._injected_flights + ], + ) + raise RuntimeError("High-severity issue did not abort test scenario") + + config = self._evaluation_configuration.configuration + self._virtual_observer = VirtualObserver( + injected_flights=InjectedFlightCollection(self._injected_flights), + repeat_query_rect_period=config.repeat_query_rect_period, + min_query_diagonal_m=config.min_query_diagonal, + relevant_past_data_period=self._rid_version.realtime_period + + config.max_propagation_latency.timedelta, + ) + + def _poll_unauthenticated_during_flights(self): + config = self._evaluation_configuration.configuration + + # Evaluate observed RID system states + evaluator = display_data_evaluator.RIDObservationEvaluator( + self, + self._injected_flights, + config, + self._rid_version, + self._dss_pool.dss_instances[0] if self._dss_pool else None, + ) + + t_end = self._virtual_observer.get_last_time_of_interest() + t_now = arrow.utcnow() + + if t_now > t_end: + raise RuntimeError( + f"Cannot evaluate RID system: injected test flights ended at {t_end}, which is before now ({t_now})" + ) + + logger.debug(f"Polling from {t_now} until {t_end}") + for f in self._injected_flights: + span = f.flight.get_span() + logger.debug( + f"Flight {f.uss_participant_id}/{f.flight.injection_id} {span[0].isoformat()} to {span[1].isoformat()}", + ) + + t_next = arrow.utcnow() + dt = config.min_polling_interval.timedelta + while arrow.utcnow() < t_end: + # Evaluate the system at an instant in time for various areas + diagonals_m = [ + self._rid_version.max_diagonal_km * 1000 + 500, # too large + self._rid_version.max_diagonal_km * 1000 - 100, # clustered + self._rid_version.max_details_diagonal_km * 1000 - 100, # details + ] + auth_tests = [] + for diagonal_m in diagonals_m: + rect = self._virtual_observer.get_query_rect(diagonal_m) + auth_tests.append(evaluator.evaluate_and_test_authentication(rect)) + + # If we checked for all diagonals that flights queries are properly authenticated, + # we can stop polling + if all(auth_tests): + self.record_note( + "unauthenticated_polling", + "Authentication check is complete, ending polling now.", + ) + break + + # Wait until minimum polling interval elapses + while t_next < arrow.utcnow(): + t_next += dt + if t_next > t_end: + break + delay = t_next - arrow.utcnow() + if delay.total_seconds() > 0: + logger.debug( + f"Waiting {delay.total_seconds()} seconds before polling RID system again..." + ) + time.sleep(delay.total_seconds()) + + def cleanup(self): + self.begin_cleanup() + while self._injected_tests: + injected_test = self._injected_tests.pop() + matching_sps = [ + sp + for sp in self._service_providers.service_providers + if sp.participant_id == injected_test.participant_id + ] + if len(matching_sps) != 1: + matching_ids = ", ".join(sp.participant_id for sp in matching_sps) + raise RuntimeError( + f"Found {len(matching_sps)} service providers with participant ID {injected_test.participant_id} ({matching_ids}) when exactly 1 was expected" + ) + sp = matching_sps[0] + check = self.check("Successful test deletion", [sp.participant_id]) + try: + query = sp.delete_test(injected_test.test_id, injected_test.version) + self.record_query(query) + if query.status_code != 200: + raise ValueError( + f"Received status code {query.status_code} after attempting to delete test {injected_test.test_id} at version {injected_test.version} from service provider {sp.participant_id}" + ) + check.record_passed() + except (RequestException, ValueError) as e: + stacktrace = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + check.record_failed( + summary="Error while trying to delete test flight", + severity=Severity.Medium, + details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}", + ) + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index c069a0beaa..922846c35c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -6,6 +6,8 @@ import math import s2sphere from s2sphere import LatLng, LatLngRect + +from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.uss_qualifier.scenarios.astm.netrid.common_dictionary_evaluator import ( RIDCommonDictionaryEvaluator, ) @@ -216,6 +218,78 @@ def __init__( f"Cannot evaluate a system using RID version {rid_version} with a DSS using RID version {dss.rid_version}" ) + def evaluate_and_test_authentication( + self, + rect: s2sphere.LatLngRect, + ) -> bool: + """Queries all flights in the expected way, then repeats the queries to SPs without credentials. + + returns true once queries to SPS have been been made without. + """ + if not self._dss: + # TODO too stringent? + raise ValueError( + "Cannot test authentication without a DSS: it is required to provide us with the flights URL of the SP(s) under test" + ) + + with self._test_scenario.check("Missing credentials") as check: + # We grab all flights from the SP's. This is authenticated + # and is expected to succeed + sp_observation = all_flights( + rect, + include_recent_positions=True, + get_details=True, + rid_version=self._rid_version, + session=self._dss.client, + ) + # We fish out the queries that were used to grab the flights from the SP, + # and attempt to re-query without credentials. This should fail. + + unauthenticated_session = UTMClientSession( + prefix_url=self._dss.client.get_prefix_url(), + auth_adapter=None, + timeout_seconds=self._dss.client.timeout_seconds, + ) + + queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list( + sp_observation.uss_flight_details_queries.values() + ) + + if len(queries_to_repeat) == 0: + self._test_scenario.record_note( + "unauthenticated_queries", + "no flights queries to repeat at this point.", + ) + return False + + self._test_scenario.record_note( + "unauthenticated_queries", + f"about to repeat {len(queries_to_repeat)} flights queries without credentials", + ) + + # Attempt to re-query the flights and flight details URLs: + for fq in list(sp_observation.uss_flight_queries.values()) + list( + sp_observation.uss_flight_details_queries.values() + ): + failed_q = fetch.query_and_describe( + unauthenticated_session, + "GET", + fq.query.request.url, + ) + if failed_q.response.code == 200: + check.record_failed( + "unauthenticated request was fulfilled", + participants=[ + fq.query.get("server_id", "unknown") + ], # TODO confirm server id is set here + severity=Severity.MEDIUM, + details=f"queried flights on {fq.query.request.url} with no credentials, expected a failure but got a success reply", + ) + # Keep track of the failed queries, too + self._test_scenario.record_query(failed_q) + + return True + def evaluate_system_instantaneously( self, observers: List[RIDSystemObserver], @@ -232,6 +306,7 @@ def evaluate_system_instantaneously( rid_version=self._rid_version, session=self._dss.client, ) + for q in sp_observation.queries: self._test_scenario.record_query(q) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py index 1552d4ee7d..ab08466a88 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py @@ -1,3 +1,4 @@ from .dss_interoperability import DSSInteroperability from .nominal_behavior import NominalBehavior +from .misbehavior import Misbehavior from .aggregate_checks import AggregateChecks diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md new file mode 100644 index 0000000000..e226494cb6 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -0,0 +1,47 @@ +# ASTM NetRID SP clients misbehavior handling test scenario + +## Overview + +In this scenario, the service provider's endpoint are accessed directly with missing or incorrect credentials. Resources that exists as well as resources that are not expected to exist are queried. + +## Resources + +### service_providers + +A set of [`NetRIDServiceProviders`](../../../../resources/netrid/service_providers.py) to be tested for proper request authentication. This scenario requires at least one SP under test. + +### evaluation_configuration + +This [`EvaluationConfigurationResource`](../../../../resources/netrid/evaluation.py) defines how to gauge success when observing the injected flights. + +## Unauthenticated requests test case + +### Injection test step + +In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. + +#### Successful injection check + +Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. + +**[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. +This check will fail if the flight was not successfully injected. + +#### Identifiable flights check + +This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. + +### Unauthenticated requests test step + +#### Missing credentials check + +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)**, +and that requests for existing flights that are executed with missing or incorrect credentials fail. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove injected data from all SPs. + +### Successful test deletion check + +**[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py new file mode 100644 index 0000000000..2c8dd182ea --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid.common.misbehavior import ( + Misbehavior as CommonMisbehavior, +) + + +class Misbehavior(TestScenario, CommonMisbehavior): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 803088ea41..a9f943cb72 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -100,6 +100,10 @@ The timestamps of the injected telemetry usually start in the future. If a flig **[astm.f3411.v19.NET0430](../../../../requirements/astm/f3411/v19.md)** requires that a NetRID Display Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. +#### Missing credentials check + +A service provider returning any flight data for queries that do not contain the correct credential violates **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**. + ### Observer polling test step In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. @@ -162,6 +166,10 @@ Taking into account the propagation time of the injected flights, if the total n For a display area with a diagonal greather than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v19.NET0480](../../../../requirements/astm/f3411/v19.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area covering no less than *NetMinClusterSize* percent of the display area size. This check validates that the display area of a cluster, measured and provided in square meters by the test harness, is no less than *NetMinClusterSize* percent of the display area. +#### Missing credentials check + +A service provider returning any flight data for queries that do not contain the correct credential violates **[astm.f3411.v19.NET0210](../../../../requirements/astm/f3411/v19.md)**. + ## Cleanup The cleanup phase of this test scenario attempts to remove injected data from all SPs. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md new file mode 100644 index 0000000000..a139d181c4 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -0,0 +1,43 @@ +# ASTM NetRID SP clients misbehavior handling test scenario + +## Overview + +In this scenario, the service provider's endpoint are accessed directly with missing or incorrect credentials. Resources that exists as well as resources that are not expected to exist are queried. + +## Resources + +### service_providers + +A set of [`NetRIDServiceProviders`](../../../../resources/netrid/service_providers.py) to be tested for proper request authentication. This scenario requires at least one SP under test. + +### evaluation_configuration + +This [`EvaluationConfigurationResource`](../../../../resources/netrid/evaluation.py) defines how to gauge success when observing the injected flights. + +## Unauthenticated requests test case + +### Injection test step + +In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. + +#### Successful injection check + +Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. + +**[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. +This check will fail if the flight was not successfully injected. + +### Unauthenticated requests test step + +#### Credential authentication check + +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)**, +and that requests for existing flights that are executed with missing or incorrect credentials fail. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove injected data from all SPs. + +### Successful test deletion check + +**[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py new file mode 100644 index 0000000000..2c8dd182ea --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid.common.misbehavior import ( + Misbehavior as CommonMisbehavior, +) + + +class Misbehavior(TestScenario, CommonMisbehavior): + pass diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index b91c6ea824..57f4b030c9 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -7,3 +7,4 @@ 1. Action generator: `action_generators.astm.f3411.ForEachDSS` 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) +4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml index 930fd79b59..7d19288925 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml @@ -50,6 +50,15 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.Misbehavior + resources: + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + dss_pool: dss_instances + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v19.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 8dd20db5b5..d3ecfe59ec 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -7,3 +7,4 @@ 1. Action generator: `action_generators.astm.f3411.ForEachDSS` 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) +4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml index f113125c71..20b8715831 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml @@ -58,6 +58,15 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.Misbehavior + resources: + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + dss_pool: dss_instances + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v22a.AggregateChecks resources: