From ea12c808d423348c5e605762c82fdc7a911bfa05 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Wed, 13 Sep 2023 07:48:40 +0200 Subject: [PATCH] PR feedback --- monitoring/monitorlib/fetch/__init__.py | 15 +- monitoring/monitorlib/infrastructure.py | 3 + .../action_generators/action_generator.py | 80 +++- .../astm/f3411/for_each_dss.py | 14 + .../action_generators/definitions.py | 2 +- .../documentation/__init__.py | 0 .../documentation/definitions.py | 40 ++ .../documentation/documentation.py | 78 ++++ .../flight_planning/planner_combinations.py | 24 +- .../action_generators/repetition/repeat.py | 14 + .../configurations/dev/uspace.yaml | 2 +- monitoring/uss_qualifier/reports/report.py | 6 +- .../interuss/automated_testing/rid.md | 10 - .../automated_testing/rid/injection.md | 12 +- .../automated_testing/rid/observation.md | 8 +- .../interuss/uss_qualifier/unit_test.md | 4 +- monitoring/uss_qualifier/scenarios/README.md | 2 + .../astm/netrid/common/aggregate_checks.py | 31 +- .../common/dss/subscription_validation.py | 101 +++-- .../astm/netrid/common/misbehavior.py | 342 ++++++++++++++ .../astm/netrid/common/nominal_behavior.py | 26 +- .../astm/netrid/display_data_evaluator.py | 8 +- .../scenarios/astm/netrid/v19/__init__.py | 1 + .../astm/netrid/v19/aggregate_checks.md | 27 +- .../netrid/v19/dss/subscription_validation.md | 72 ++- .../astm/netrid/v19/dss_interoperability.md | 4 + .../scenarios/astm/netrid/v19/misbehavior.md | 60 +++ .../scenarios/astm/netrid/v19/misbehavior.py | 11 + .../astm/netrid/v19/nominal_behavior.md | 2 + .../astm/netrid/v19/nominal_behavior.py | 5 +- .../scenarios/astm/netrid/v22a/__init__.py | 1 + .../astm/netrid/v22a/aggregate_checks.md | 27 +- .../astm/netrid/v22a/dss/isa_simple.md | 8 +- .../v22a/dss/subscription_validation.md | 72 ++- .../astm/netrid/v22a/dss_interoperability.md | 4 + .../scenarios/astm/netrid/v22a/misbehavior.md | 60 +++ .../scenarios/astm/netrid/v22a/misbehavior.py | 11 + .../astm/netrid/v22a/nominal_behavior.md | 2 + .../astm/netrid/v22a/nominal_behavior.py | 5 +- .../uss_qualifier/scenarios/definitions.py | 8 +- .../scenarios/documentation/definitions.py | 1 + .../scenarios/documentation/parsing.py | 17 +- .../general_flight_authorization.py | 1 + .../geospatial_feature_comprehension.py | 1 + .../uss_qualifier/scenarios/scenario.py | 19 +- .../suites/astm/netrid/f3411_19.md | 264 ++++++++++- .../suites/astm/netrid/f3411_19.yaml | 9 + .../astm/netrid/f3411_19/dss_probing.md | 27 ++ .../suites/astm/netrid/f3411_19_suite1.md | 34 ++ .../suites/astm/netrid/f3411_22a.md | 309 ++++++++++++- .../suites/astm/netrid/f3411_22a.yaml | 9 + .../astm/netrid/f3411_22a/dss_probing.md | 37 ++ .../suites/astm/netrid/f3411_22a_suite1.md | 40 ++ .../uss_qualifier/suites/astm/utm/f3548_21.md | 127 +++++- .../suites/documentation/documentation.py | 429 ++++++++++++++++-- .../documentation/format_documentation.py | 27 +- .../suites/faa/uft/message_signing.md | 118 +++++ .../suites/uspace/flight_auth.md | 121 ++++- .../suites/uspace/geo_awareness_cis.md | 4 + .../suites/uspace/network_identification.md | 304 +++++++++++++ .../suites/uspace/required_services.md | 411 +++++++++++++++++ .../definitions/TestScenarioDeclaration.json | 36 +- 62 files changed, 3318 insertions(+), 229 deletions(-) create mode 100644 monitoring/uss_qualifier/action_generators/documentation/__init__.py create mode 100644 monitoring/uss_qualifier/action_generators/documentation/definitions.py create mode 100644 monitoring/uss_qualifier/action_generators/documentation/documentation.py delete mode 100644 monitoring/uss_qualifier/requirements/interuss/automated_testing/rid.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py create mode 100644 monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md create mode 100644 monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 28a5615c50..1b7815bc22 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -269,12 +269,15 @@ def describe_query( query_type: Optional[QueryType] = None, server_id: Optional[str] = None, ) -> Query: - return Query( + query = Query( request=describe_request(resp.request, initiated_at), response=describe_response(resp), - server_id=server_id, - query_type=query_type, ) + if query_type is not None: + query.query_type = query_type + if server_id is not None: + query.server_id = server_id + return query def query_and_describe( @@ -282,6 +285,7 @@ def query_and_describe( verb: str, url: str, query_type: Optional[QueryType] = None, + server_id: Optional[str] = None, **kwargs, ) -> Query: """Attempt to perform a query, and then describe the results of that attempt. @@ -305,7 +309,6 @@ def query_and_describe( else: utm_session = True req_kwargs = kwargs.copy() - req_kwargs.pop("server_id", None) if "timeout" not in req_kwargs: req_kwargs["timeout"] = TIMEOUTS @@ -320,7 +323,7 @@ def query_and_describe( client.request(verb, url, **req_kwargs), t0, query_type=query_type, - server_id=kwargs.get("server_id", None), + server_id=server_id, ) except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}" @@ -350,7 +353,7 @@ def query_and_describe( elapsed_s=(t1 - t0).total_seconds(), reported=StringBasedDateTime(t1), ), - server_id=kwargs.get("server_id", None), + server_id=server_id, ) if query_type is not None: result.query_type = query_type diff --git a/monitoring/monitorlib/infrastructure.py b/monitoring/monitorlib/infrastructure.py index 1e789d3263..5c8f1de086 100644 --- a/monitoring/monitorlib/infrastructure.py +++ b/monitoring/monitorlib/infrastructure.py @@ -133,6 +133,9 @@ def request(self, method, url, **kwargs): return super().request(method, url, **kwargs) + def get_prefix_url(self): + return self._prefix_url + class AsyncUTMTestSession: """ diff --git a/monitoring/uss_qualifier/action_generators/action_generator.py b/monitoring/uss_qualifier/action_generators/action_generator.py index 71694d5547..5505dd8d04 100644 --- a/monitoring/uss_qualifier/action_generators/action_generator.py +++ b/monitoring/uss_qualifier/action_generators/action_generator.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod import inspect -from typing import Generic, Dict, Optional, TypeVar +from typing import Generic, Dict, Optional, TypeVar, List, Type from implicitdict import ImplicitDict from monitoring import uss_qualifier as uss_qualifier_module @@ -12,6 +12,10 @@ from monitoring.uss_qualifier.action_generators.definitions import ( ActionGeneratorSpecificationType, ActionGeneratorDefinition, + GeneratorTypeName, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, ) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID @@ -45,32 +49,35 @@ def run_next_action(self) -> Optional[TestSuiteActionReport]: "A concrete action generator must implement `actions` method" ) + @classmethod + def list_potential_actions( + cls, specification: Optional[ActionGeneratorSpecificationType] + ) -> List[PotentialGeneratedAction]: + """Enumerate the potential actions that may be run by an instance of this generator type. + + Concrete subclasses of ActionGenerator must implement a classmethod that shadows this one according to this + specification. + + Args: + specification: A serializable (subclass of implicitdict.ImplicitDict) specification for how to create the + action generator instance, or None if the action generator type does not need a specification. + + Returns: All potential actions that may be generated by this generator, depending on the resources provided. + """ + raise NotImplementedError( + "A concrete action generator must implement `list_potential_actions` classmethod" + ) + @staticmethod def make_from_definition( definition: ActionGeneratorDefinition, resources: Dict[ResourceID, ResourceType] ) -> ActionGeneratorType: - from monitoring.uss_qualifier import ( - action_generators as action_generators_module, + action_generator_type = action_generator_type_from_name( + definition.generator_type ) + specification_type = action_generator_specification_type(action_generator_type) - import_submodules(action_generators_module) - action_generator_type = get_module_object_by_name( - parent_module=uss_qualifier_module, - object_name=definition.generator_type, - ) - if not issubclass(action_generator_type, ActionGenerator): - raise NotImplementedError( - "Action generator type {} is not a subclass of the ActionGenerator base class".format( - action_generator_type.__name__ - ) - ) - constructor_signature = inspect.signature(action_generator_type.__init__) - specification_type = None constructor_args = {} - for arg_name, arg in constructor_signature.parameters.items(): - if arg_name == "specification": - specification_type = arg.annotation - break if specification_type is not None: constructor_args["specification"] = ImplicitDict.parse( definition.specification, specification_type @@ -82,3 +89,36 @@ def make_from_definition( ActionGeneratorType = TypeVar("ActionGeneratorType", bound=ActionGenerator) + + +def action_generator_type_from_name( + action_generator_type_name: GeneratorTypeName, +) -> Type[ActionGenerator]: + from monitoring.uss_qualifier import ( + action_generators as action_generators_module, + ) + + import_submodules(action_generators_module) + action_generator_type = get_module_object_by_name( + parent_module=uss_qualifier_module, + object_name=action_generator_type_name, + ) + if not issubclass(action_generator_type, ActionGenerator): + raise NotImplementedError( + "Action generator type {} is not a subclass of the ActionGenerator base class".format( + action_generator_type.__name__ + ) + ) + return action_generator_type + + +def action_generator_specification_type( + action_generator_type: Type[ActionGenerator], +) -> Optional[Type]: + constructor_signature = inspect.signature(action_generator_type.__init__) + specification_type = None + for arg_name, arg in constructor_signature.parameters.items(): + if arg_name == "specification": + specification_type = arg.annotation + break + return specification_type diff --git a/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py b/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py index 95c756d4c0..bbbee1a469 100644 --- a/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py +++ b/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py @@ -3,6 +3,12 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.astm.f3411 import ( DSSInstanceResource, @@ -34,6 +40,14 @@ class ForEachDSS(ActionGenerator[ForEachDSSSpecification]): _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: ForEachDSSSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: ForEachDSSSpecification, diff --git a/monitoring/uss_qualifier/action_generators/definitions.py b/monitoring/uss_qualifier/action_generators/definitions.py index 8533a5ccc1..aca51c8bfa 100644 --- a/monitoring/uss_qualifier/action_generators/definitions.py +++ b/monitoring/uss_qualifier/action_generators/definitions.py @@ -5,7 +5,7 @@ GeneratorTypeName = str -"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier.action_generators` module""" +"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier` module""" ActionGeneratorSpecificationType = TypeVar( diff --git a/monitoring/uss_qualifier/action_generators/documentation/__init__.py b/monitoring/uss_qualifier/action_generators/documentation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/action_generators/documentation/definitions.py b/monitoring/uss_qualifier/action_generators/documentation/definitions.py new file mode 100644 index 0000000000..33bb951e80 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/documentation/definitions.py @@ -0,0 +1,40 @@ +from typing import Optional + +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName +from monitoring.uss_qualifier.fileio import FileReference +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName +from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition + + +class PotentialTestScenarioAction(ImplicitDict): + scenario_type: TestScenarioTypeName + """Type of test scenario.""" + + +class PotentialTestSuiteAction(ImplicitDict): + suite_type: Optional[FileReference] + """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" + + suite_definition: Optional[TestSuiteDefinition] + """Definition of test suite internal to the configuration -- specified instead of `suite_type`.""" + + +class PotentialActionGeneratorAction(ImplicitDict): + generator_type: GeneratorTypeName + """Type of action generator.""" + + specification: dict + """Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`""" + + +class PotentialGeneratedAction(ImplicitDict): + test_scenario: Optional[PotentialTestScenarioAction] + test_suite: Optional[PotentialTestSuiteAction] + action_generator: Optional[PotentialActionGeneratorAction] + + def get_action_type(self) -> ActionType: + matches = [v for v in ActionType if v in self and self[v]] + if len(matches) != 1: + ActionType.raise_invalid_action_declaration() + return ActionType(matches[0]) diff --git a/monitoring/uss_qualifier/action_generators/documentation/documentation.py b/monitoring/uss_qualifier/action_generators/documentation/documentation.py new file mode 100644 index 0000000000..ec1193f4c5 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/documentation/documentation.py @@ -0,0 +1,78 @@ +from typing import List, Union + +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, + action_generator_specification_type, +) +from monitoring.uss_qualifier.action_generators.definitions import ( + ActionGeneratorDefinition, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, + PotentialTestSuiteAction, + PotentialActionGeneratorAction, +) +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteActionDeclaration, + ActionType, +) + + +def list_potential_actions_for_action_generator_definition( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction] +) -> List[PotentialGeneratedAction]: + action_generator_type = action_generator_type_from_name( + generator_def.generator_type + ) + specification_type = action_generator_specification_type(action_generator_type) + if specification_type is not None: + spec = ImplicitDict.parse(generator_def.specification, specification_type) + else: + spec = None + return action_generator_type.list_potential_actions(spec) + + +def list_potential_actions_for_action_declaration( + declaration: TestSuiteActionDeclaration, +) -> List[PotentialGeneratedAction]: + action_type = declaration.get_action_type() + if action_type == ActionType.TestScenario: + return [ + PotentialGeneratedAction( + test_scenario=PotentialTestScenarioAction( + scenario_type=declaration.test_scenario.scenario_type + ) + ) + ] + elif action_type == ActionType.TestSuite: + if "suite_type" in declaration.test_suite and declaration.test_suite.suite_type: + return [ + PotentialGeneratedAction( + test_suite=PotentialTestSuiteAction( + suite_type=declaration.test_suite.suite_type + ) + ) + ] + elif ( + "suite_definition" in declaration.test_suite + and declaration.test_suite.suite_definition + ): + return [ + PotentialGeneratedAction( + test_suite=PotentialTestSuiteAction( + suite_definition=declaration.test_suite.suite_definition + ) + ) + ] + elif action_type == ActionType.ActionGenerator: + return [ + PotentialGeneratedAction( + action_generator=PotentialActionGeneratorAction( + generator_type=declaration.action_generator.generator_type + ) + ) + ] + else: + raise NotImplementedError(f"Action type {action_type} is not supported") diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py index f81899c677..38d2ddf54e 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py +++ b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py @@ -3,6 +3,17 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, + action_generator_specification_type, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource @@ -14,7 +25,10 @@ ResourceType, ) -from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteActionDeclaration, + ActionType, +) from monitoring.uss_qualifier.suites.suite import ( ActionGenerator, TestSuiteAction, @@ -43,6 +57,14 @@ class FlightPlannerCombinations( _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: FlightPlannerCombinationsSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: FlightPlannerCombinationsSpecification, diff --git a/monitoring/uss_qualifier/action_generators/repetition/repeat.py b/monitoring/uss_qualifier/action_generators/repetition/repeat.py index c282a73693..a2f3256af2 100644 --- a/monitoring/uss_qualifier/action_generators/repetition/repeat.py +++ b/monitoring/uss_qualifier/action_generators/repetition/repeat.py @@ -1,6 +1,12 @@ from typing import Dict, List, Optional from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.resource import ResourceType @@ -26,6 +32,14 @@ class Repeat(ActionGenerator[RepeatSpecification]): _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: RepeatSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: RepeatSpecification, diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index c21974ae80..71626807f3 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -29,5 +29,5 @@ v1: report: report_path: output/report_uspace.json templated_reports: - - template_url: https://github.com/Orbitalize/reports/releases/download/v0.0.14/app-v0.0.14.zip + - template_url: https://github.com/Orbitalize/reports/releases/download/v0.0.15/app-v0.0.15.zip output_path: output/capabilities_uspace.html diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index d6bb83e63c..cb4371d828 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -7,6 +7,7 @@ from implicitdict import ImplicitDict, StringBasedDateTime from monitoring.monitorlib import fetch, inspection +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ( TestConfiguration, @@ -18,6 +19,7 @@ JSONPathExpression, ) from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName class FailedCheck(ImplicitDict): @@ -211,7 +213,7 @@ class TestScenarioReport(ImplicitDict): name: str """Name of this test scenario""" - scenario_type: str + scenario_type: TestScenarioTypeName """Type of this test scenario""" documentation_url: str @@ -295,7 +297,7 @@ def participant_ids(self) -> Set[ParticipantID]: class ActionGeneratorReport(ImplicitDict): - generator_type: str + generator_type: GeneratorTypeName """Type of action generator""" actions: List[TestSuiteActionReport] diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid.md deleted file mode 100644 index 0c5725a6f7..0000000000 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid.md +++ /dev/null @@ -1,10 +0,0 @@ -# InterUSS RemoteID Testing Requirements - -## Overview - -TODO: Describe overall approach, link to injection & observation - -## Requirements - -TODO: Describe requirements - diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/injection.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/injection.md index c653a5a92d..f3f3590e2c 100644 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/injection.md +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/injection.md @@ -2,16 +2,24 @@ ## Overview -TODO: Link to API YAML and provide overview +In order to test remote ID Service Providers (USSs who ingest aircraft data from operators and provide it to relevant Display Providers), InterUSS requires any Service Provider under test to implement the [InterUSS remote ID automated testing interface](https://github.com/interuss/automated_testing_interfaces/tree/main/rid) (specifically, the [injection portion](https://github.com/interuss/automated_testing_interfaces/blob/main/rid/v1/injection.yaml)). This interface empowers uss_qualifier, as the test director, to instruct the USS to create a flight in a particular area with particular characteristics. This is analogous to a similar verbal instruction during a manual checkout (e.g., "USS X, start your flight in the yellow area"). ## Requirements -TODO: Describe requirements +In general, to be successfully tested by uss_qualifier for remote ID functionality, Service Provider USSs are expected to implement the injection API mentioned above and successfully respond to valid requests. ### UpsertTestSuccess +When a properly-authorized valid request to create/start/simulate a set of flights ("test"), the USS under test must execute this request (create/start/simulate the flights) and indicate success. + ### UpsertTestResult +The USS under test is allowed to modify most components of the requested flights including the telemetry and details responses, including the flight ID (though not the injection_id; that is the invariant ID by which uss_qualifier refers to an injected test flight). However, there are obviously limits on how much the injected flight may be modified and still serve the purposes of the test being conducted. For instance, a test whose purpose was to observe interactions between multiple providers' flights in the same area could not be accomplished if all but one USS created flights in a particular city in Europe while the last USS created their flight in the United States. The actually-injected flight returned by the USS under test in response to an injection request must be suitable for the test to be conducted. In practice, this means that USSs should modify the injection request as little as possible while still maintaining compatibility with their system. + ### ExpectedBehavior +The USS under test must treat injected flights as similarly to real flights as practical. The same pathways that accept real telemetry should be used to accept the injected telemetry. For instance, the standard pathway to accept real telemetry should generally not accept reports from the future, so if future telemetry of the injected flight is visible appreciably before the time it is supposed to have been virtually reported then the USS into which the flight was injected is not treating the injected flight the same way a normal flight would be treated. + ### DeleteTestSuccess + +In order to rapidly conduct sequences of automated tests, uss_qualifier (as test director) must be able to "clear the airspace" after the completion of a test so that the simulated/injected flights from this test run do not affect future test runs. A Service Provider USS must successfully cancel/land/remove/delete the flights in a specified test upon deletion request for that test and indicate success for the deletion request. diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/observation.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/observation.md index 2073090de5..1e9f4da48c 100644 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/observation.md +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/rid/observation.md @@ -2,12 +2,16 @@ ## Overview -TODO: Link to API YAML and provide overview +In order to test remote ID Display Providers (USSs who obtain aircraft data from Service Providers in order to aggregate it for their end-user viewers), InterUSS requires any Display Provider under test to implement the [InterUSS remote ID automated testing interface](https://github.com/interuss/automated_testing_interfaces/tree/main/rid) (specifically, the [observation portion](https://github.com/interuss/automated_testing_interfaces/blob/main/rid/v1/observation.yaml)). This interface empowers uss_qualifier, as the test director, to ask the USS what flights would be visible on its Display Application(s). This is analogous to a similar verbal question during a manual checkout (e.g., "USS Y, what flights do you see in the yellow area?"). ## Requirements -TODO: Describe requirements +In general, to be successfully tested by uss_qualifier for remote ID functionality, Display Provider USSs are expected to implement the observation API mentioned above and successfully respond to valid requests. ### ObservationSuccess +Upon receipt of a properly-authorized valid request to report the flights visible in a particular area, the USS under test must respond with the content visible in that area. Even if the user/viewer would be shown an error or other off-nominal situation in the Display Application, it should still be possible to return a valid response to this question (e.g., in the case of the user being shown an error, the USS would indicate no flights and no cluster visible). + ### UniqueFlights + +While the identifiers for displayed flights are an implementation detail of the USS (it may use NetRID identifiers, or it may assign different identifiers), the user/viewer needs the capability to obtain details about a specific individual flight. For that reason, a USS must provide unique identifiers for each flight in a particular observation. diff --git a/monitoring/uss_qualifier/requirements/interuss/uss_qualifier/unit_test.md b/monitoring/uss_qualifier/requirements/interuss/uss_qualifier/unit_test.md index bd04f6f06b..8693e0a333 100644 --- a/monitoring/uss_qualifier/requirements/interuss/uss_qualifier/unit_test.md +++ b/monitoring/uss_qualifier/requirements/interuss/uss_qualifier/unit_test.md @@ -1,8 +1,8 @@ # USS Qualifier unit test requirements -This file is used to defines requirements to support unit testing. +This file is used to define requirements to support unit testing. ## Dummy requirements * REQ001 * REQ100 -* REQ101 \ No newline at end of file +* REQ101 diff --git a/monitoring/uss_qualifier/scenarios/README.md b/monitoring/uss_qualifier/scenarios/README.md index 75a5986a26..09b920b3eb 100644 --- a/monitoring/uss_qualifier/scenarios/README.md +++ b/monitoring/uss_qualifier/scenarios/README.md @@ -87,6 +87,8 @@ A check should document the requirement(s) violated if the check fails. Require Any requirements identified (e.g., `**astm.f3411.v19.NET0420**`) must be documented as well. See [the requirements documentation](../requirements/README.md) for more information. +If the text of this section includes `TODO:`, then the check will be indicated as in development rather than complete. Documentation of intended checks with, e.g., `TODO: Implement` prior to the start of Python development is highly encouraged. + ### Cleanup phase If a test scenario wants to perform a cleanup procedure follow any non-error termination of the rest of the scenario, it must: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index a800e9f019..1081fc1324 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -22,6 +22,8 @@ NetRIDServiceProvider, ) +from loguru import logger + class AggregateChecks(ReportEvaluationScenario): _rid_version: RIDVersion @@ -141,7 +143,6 @@ def run(self): self.end_test_scenario() def _verify_https_everywhere(self): - for participant_id, participant_queries in self._queries_by_participant.items(): self._inspect_participant_queries(participant_id, participant_queries) @@ -155,12 +156,9 @@ def _verify_https_everywhere(self): # TODO clean this up: this is an internal requirement and not a check, # leaving as-is during development to make sure the test-suite runs but we know about unattributed queries # ultimately this check could go into the constructor and blow things up early - self.record_note( - "unattributed_queries", - f"found {len(unattr_queries)} unattributed queries", - ) + with self.check( - "All interactions happen over https", + "No unattributed queries", [], ) as check: check.record_failed( @@ -170,13 +168,12 @@ def _verify_https_everywhere(self): def _inspect_participant_queries( self, participant_id: str, participant_queries: List[fetch.Query] ): - found_cleartext_query = False + cleartext_queries = [] for query in participant_queries: if query.request.url.startswith("http://"): - found_cleartext_query = True + cleartext_queries.append(query) if participant_id not in self._debug_mode_usses: - self.record_note( - "cleartext-query", + logger.info( f"query is not https: {participant_id} - {query.request.url}", ) @@ -185,9 +182,17 @@ def _inspect_participant_queries( "All interactions happen over https", [participant_id], ) as check: - if found_cleartext_query: + if len(cleartext_queries) > 0: + timestamps = [ + q.request.initiated_at.datetime for q in cleartext_queries + ] + urls = set([q.request.url for q in cleartext_queries]) check.record_failed( - "found at least one cleartext http query", Severity.Medium + summary=f"found {len(cleartext_queries)} cleartext http queries", + details=f"unique cleartext urls: {urls}", + severity=Severity.Medium, + participants=[participant_id], + query_timestamps=timestamps, ) else: self.record_note( @@ -203,7 +208,6 @@ def _dp_display_data_details_times_step(self): for participant, all_queries in self._queries_by_participant.items(): relevant_queries: List[fetch.Query] = list() for query in all_queries: - if ( query.status_code == 200 and query.has_field_with_value("query_type") @@ -303,7 +307,6 @@ def _dp_display_data_times_step(self): pattern = re.compile(r"/display_data\?view=(-?\d+(.\d+)?,){3}-?\d+(.\d+)?") for participant, all_queries in self._queries_by_participant.items(): - # identify successful display_data queries relevant_queries: List[fetch.Query] = list() for query in all_queries: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py index 6aa7f45c44..1c89883739 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py @@ -59,7 +59,13 @@ def run(self): self._setup_case() - self._subscription_limitations_case() + self.begin_test_case("Subscription duration limitations") + self._create_too_long_subscription() + self.end_test_case() + + self.begin_test_case("Subscription quantity limitations") + self._create_too_many_subscriptions() + self.end_test_case() self.end_test_scenario() @@ -70,46 +76,33 @@ def _setup_case(self): self.end_test_case() - def _clean_any_sub(self, check: PendingCheck): - fetched = self._dss_wrapper.search_subs( - check, self._isa.footprint.to_vertices() - ) + def _clean_any_sub(self): + with self.check( + "Successful subscription query", [self._dss.participant_id] + ) as check: + fetched = self._dss_wrapper.search_subs( + check, self._isa.footprint.to_vertices() + ) for sub_id in fetched.subscriptions.keys(): - self._dss_wrapper.cleanup_sub(check, sub_id=sub_id) + with self.check( + "Successful subscription deletion", [self._dss.participant_id] + ) as check: + self._dss_wrapper.cleanup_sub(check, sub_id=sub_id) def _ensure_clean_workspace_step(self): self.begin_test_step("Ensure clean workspace") - with self.check( - "Successful subscription query and cleanup", [self._dss.participant_id] - ) as check: - self._clean_any_sub(check) + self._clean_any_sub() self.end_test_step() - def _subscription_limitations_case(self): - self.begin_test_case("Subscription limitations") - - self.begin_test_step("Subscription duration limitations") - - self._create_too_long_subscription() - - self.end_test_step() - - self.begin_test_step("Subscription quantity limitations") - - self._create_too_many_subscriptions() - - self.end_test_step() - self.end_test_case() - def _create_too_many_subscriptions(self): + self.begin_test_step("Create maximum number of subscriptions") with self.check( "Create up to the maximum allowed number of subscriptions in an area", [self._dss.participant_id], ) as check: - self._clean_any_sub(check) # Create 10 subscriptions with different ID's for i in range(self._dss.rid_version.dss_max_subscriptions_per_area): sub_id = f"{self._sub_id[:-3]}1{i:02d}" @@ -119,6 +112,10 @@ def _create_too_many_subscriptions(self): **self._default_subscription_params(datetime.timedelta(minutes=30)), ) + self.end_test_step() + + self.begin_test_step("Exceed maximum number of subscriptions") + with self.check( "Enforce maximum number of subscriptions for an area", [self._dss.participant_id], @@ -133,12 +130,21 @@ def _create_too_many_subscriptions(self): **self._default_subscription_params(datetime.timedelta(minutes=30)), ) + self.end_test_step() + + self.begin_test_step("Clean up subscriptions") + + self._clean_any_sub() + + self.end_test_step() + def _create_too_long_subscription(self): + self.begin_test_step("Try to create too-long subscription") + with self.check( - "Enforce maximum duration of subscriptions for an area", + "Too-long subscription creation rejected", [self._dss.participant_id], ) as check: - self._clean_any_sub(check) # Sub with this ID does not exist and too long: we expect either a failure, or # that any subscription that is effectively created to be truncated at 24 hours. creation_attempt = self._dss_wrapper.put_sub_expect_response_code( @@ -156,8 +162,16 @@ def _create_too_long_subscription(self): if creation_attempt.success: self._check_properly_truncated(check, creation_attempt) + self.end_test_step() + + self.begin_test_step("Try to extend subscription") + + with self.check( + "Valid subscription created", + [self._dss.participant_id], + ) as check: # Create a subscription that is fine - self._dss_wrapper.put_sub( + sub = self._dss_wrapper.put_sub( check=check, sub_id=self._sub_id, **self._default_subscription_params( @@ -165,7 +179,11 @@ def _create_too_long_subscription(self): ), ) - # Sub with this ID does exist, an we try to extend it beyond 24 hours: + with self.check( + "Subscription duration limited during update", + [self._dss.participant_id], + ) as check: + # Sub with this ID does exist, and we try to extend it beyond 24 hours: extended_subscription = self._dss_wrapper.put_sub_expect_response_code( check=check, sub_id=self._sub_id, @@ -177,6 +195,20 @@ def _create_too_long_subscription(self): if extended_subscription.success: self._check_properly_truncated(check, extended_subscription) + self.end_test_step() + + self.begin_test_step("Remove subscription") + + with self.check( + "Subscription deleted", + [self._dss.participant_id], + ) as check: + self._dss_wrapper.del_sub( + check=check, sub_id=self._sub_id, sub_version=sub.subscription.version + ) + + self.end_test_step() + def _check_properly_truncated( self, check: PendingCheck, changed: ChangedSubscription ): @@ -191,7 +223,7 @@ def _check_properly_truncated( Severity.Medium, f"{self._dss.participant_id} DSS instance has returned a non-properly truncated subscription " f"(duration: {duration}) " - f"when the expecation was either to fail or to truncate at 24 hours.", + f"when the expectation was either to fail or to truncate at 24 hours.", query_timestamps=[changed.query.request.timestamp], ) # If a subscription was created, we want to delete it before continuing: @@ -211,9 +243,6 @@ def _default_subscription_params(self, duration: datetime.timedelta) -> Dict: def cleanup(self): self.begin_cleanup() - with self.check( - "Successful subscription query and cleanup", [self._dss.participant_id] - ) as check: - self._clean_any_sub(check) + self._clean_any_sub() self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py new file mode 100644 index 0000000000..fe75b44ebf --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -0,0 +1,342 @@ +import time +import traceback +import uuid +from typing import List + +import arrow +import s2sphere +from implicitdict import ImplicitDict +from loguru import logger +from requests.exceptions import RequestException +from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse + +from monitoring.monitorlib import fetch +from monitoring.monitorlib.fetch import rid +from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.monitorlib.rid import RIDVersion +from monitoring.monitorlib.rid_automated_testing.injection_api import ( + CreateTestParameters, +) +from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource +from monitoring.uss_qualifier.resources.netrid import ( + FlightDataResource, + NetRIDServiceProviders, + EvaluationConfigurationResource, +) +from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator +from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( + InjectedFlightCollection, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.injection import ( + InjectedFlight, + InjectedTest, +) +from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( + VirtualObserver, +) +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + + +class Misbehavior(GenericTestScenario): + """ + Check that an unauthenticated client is not able to query a Service Provider + """ + + _flights_data: FlightDataResource + _service_providers: NetRIDServiceProviders + _evaluation_configuration: EvaluationConfigurationResource + _injected_flights: List[InjectedFlight] + _injected_tests: List[InjectedTest] + + def __init__( + self, + flights_data: FlightDataResource, + service_providers: NetRIDServiceProviders, + evaluation_configuration: EvaluationConfigurationResource, + dss_pool: DSSInstancesResource = None, + ): + super().__init__() + self._flights_data = flights_data + self._service_providers = service_providers + self._evaluation_configuration = evaluation_configuration + self._injected_flights = [] + self._injected_tests = [] + if len(dss_pool.dss_instances) == 0: + raise ValueError( + "The Misbehavior Scenario requires at least one DSS instance" + ) + self._dss = dss_pool.dss_instances[0] + + @property + def _rid_version(self) -> RIDVersion: + raise NotImplementedError( + "Misbehavior test scenario subclass must specify _rid_version" + ) + + def run(self): + self.begin_test_scenario() + self.begin_test_case("Unauthenticated requests") + + self.begin_test_step("Injection") + self._inject_flights() + self.end_test_step() + + self.begin_test_step("Unauthenticated requests") + + self._poll_unauthenticated_during_flights() + + self.end_test_step() + + self.end_test_case() + self.end_test_scenario() + + def _inject_flights(self): + # TODO consider sharing code with nominal_behavior's _inject_flights + # Inject flights into all USSs + test_id = str(uuid.uuid4()) + test_flights = self._flights_data.get_test_flights() + service_providers = self._service_providers.service_providers + if len(service_providers) > len(test_flights): + raise ValueError( + "{} service providers were specified, but data for only {} test flights were provided".format( + len(service_providers), len(test_flights) + ) + ) + for i, target in enumerate(service_providers): + p = CreateTestParameters(requested_flights=[test_flights[i]]) + check = self.check("Successful injection", [target.participant_id]) + try: + query = target.submit_test(p, test_id) + except RequestException as e: + stacktrace = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + check.record_failed( + summary="Error while trying to inject test flight", + severity=Severity.High, + details=f"While trying to inject a test flight into {target.participant_id}, encountered error:\n{stacktrace}", + ) + raise RuntimeError("High-severity issue did not abort test scenario") + self.record_query(query) + try: + if query.status_code != 200: + raise ValueError( + f"Expected response code 200 but received {query.status_code} instead" + ) + if "json" not in query.response: + raise ValueError("Response did not contain a JSON body") + changed_test: ChangeTestResponse = ImplicitDict.parse( + query.response.json, ChangeTestResponse + ) + self._injected_tests.append( + InjectedTest( + participant_id=target.participant_id, + test_id=test_id, + version=changed_test.version, + ) + ) + injections = changed_test.injected_flights + check.record_passed() + except ValueError as e: + check.record_failed( + summary="Error injecting test flight", + severity=Severity.High, + details=f"Attempting to inject a test flight into {target.participant_id}, encountered status code {query.status_code}: {str(e)}", + query_timestamps=[query.request.timestamp], + ) + raise RuntimeError("High-severity issue did not abort test scenario") + + for flight in injections: + self._injected_flights.append( + InjectedFlight( + uss_participant_id=target.participant_id, + test_id=test_id, + flight=TestFlight(flight), + query_timestamp=query.request.timestamp, + ) + ) + + # Make sure the injected flights can be identified correctly by the test harness + with self.check("Identifiable flights") as check: + errors = display_data_evaluator.injected_flights_errors( + self._injected_flights + ) + if errors: + check.record_failed( + "Injected flights not suitable for test", + Severity.High, + details="When checking the suitability of the flights (as injected) for the test, found:\n" + + "\n".join(errors), + query_timestamps=[ + f.query_timestamp for f in self._injected_flights + ], + ) + raise RuntimeError("High-severity issue did not abort test scenario") + + config = self._evaluation_configuration.configuration + self._virtual_observer = VirtualObserver( + injected_flights=InjectedFlightCollection(self._injected_flights), + repeat_query_rect_period=config.repeat_query_rect_period, + min_query_diagonal_m=config.min_query_diagonal, + relevant_past_data_period=self._rid_version.realtime_period + + config.max_propagation_latency.timedelta, + ) + + def _poll_unauthenticated_during_flights(self): + config = self._evaluation_configuration.configuration + + t_end = self._virtual_observer.get_last_time_of_interest() + t_now = arrow.utcnow() + + if t_now > t_end: + raise RuntimeError( + f"Cannot evaluate RID system: injected test flights ended at {t_end}, which is before now ({t_now})" + ) + + logger.debug(f"Polling from {t_now} until {t_end}") + for f in self._injected_flights: + span = f.flight.get_span() + logger.debug( + f"Flight {f.uss_participant_id}/{f.flight.injection_id} {span[0].isoformat()} to {span[1].isoformat()}", + ) + + t_next = arrow.utcnow() + dt = config.min_polling_interval.timedelta + while arrow.utcnow() < t_end: + # Evaluate the system at an instant in time for various areas + diagonals_m = [ + self._rid_version.max_diagonal_km * 1000 + 500, # too large + self._rid_version.max_diagonal_km * 1000 - 100, # clustered + self._rid_version.max_details_diagonal_km * 1000 - 100, # details + ] + auth_tests = [] + for diagonal_m in diagonals_m: + rect = self._virtual_observer.get_query_rect(diagonal_m) + auth_tests.append(self._evaluate_and_test_authentication(rect)) + + # If we checked for all diagonals that flights queries are properly authenticated, + # we can stop polling + if all(auth_tests): + logger.debug( + "Authentication check is complete, ending polling now.", + ) + break + + # Wait until minimum polling interval elapses + while t_next < arrow.utcnow(): + t_next += dt + if t_next > t_end: + break + delay = t_next - arrow.utcnow() + if delay.total_seconds() > 0: + logger.debug( + f"Waiting {delay.total_seconds()} seconds before polling RID system again..." + ) + time.sleep(delay.total_seconds()) + + def _evaluate_and_test_authentication( + self, + rect: s2sphere.LatLngRect, + ) -> bool: + """Queries all flights in the expected way, then repeats the queries to SPs without credentials. + + returns true once queries to SPS have been made without credentials. False otherwise, such as when + no flights were yet returned by the authenticated queries. + """ + + with self.check("Missing credentials") as check: + # We grab all flights from the SP's. This is authenticated + # and is expected to succeed + sp_observation = rid.all_flights( + rect, + include_recent_positions=True, + get_details=True, + rid_version=self._rid_version, + session=self._dss.client, + ) + # We fish out the queries that were used to grab the flights from the SP, + # and attempt to re-query without credentials. This should fail. + + unauthenticated_session = UTMClientSession( + prefix_url=self._dss.client.get_prefix_url(), + auth_adapter=None, + timeout_seconds=self._dss.client.timeout_seconds, + ) + + queries_to_repeat = list(sp_observation.uss_flight_queries.values()) + list( + sp_observation.uss_flight_details_queries.values() + ) + + if len(queries_to_repeat) == 0: + logger.debug("no flights queries to repeat at this point.") + return False + + logger.debug( + f"about to repeat {len(queries_to_repeat)} flights queries without credentials" + ) + + # Attempt to re-query the flights and flight details URLs: + for fq in queries_to_repeat: + failed_q = fetch.query_and_describe( + client=unauthenticated_session, + verb=fq.query.request.method, + url=fq.query.request.url, + json=fq.query.request.json, + data=fq.query.request.body, + ) + logger.info( + f"Repeating query to {fq.query.request.url} without credentials" + ) + server_id = fq.query.get("server_id", "unknown") + if failed_q.response.code not in [401, 403]: + check.record_failed( + "unauthenticated request was fulfilled", + participants=[server_id], + severity=Severity.MEDIUM, + details=f"queried flights on {fq.query.request.url} with no credentials, expected a failure but got a success reply", + ) + else: + logger.info( + f"participant with id {server_id} properly authenticated the request" + ) + # Keep track of the failed queries, too + self.record_query(failed_q) + + return True + + def cleanup(self): + self.begin_cleanup() + while self._injected_tests: + injected_test = self._injected_tests.pop() + matching_sps = [ + sp + for sp in self._service_providers.service_providers + if sp.participant_id == injected_test.participant_id + ] + if len(matching_sps) != 1: + matching_ids = ", ".join(sp.participant_id for sp in matching_sps) + raise RuntimeError( + f"Found {len(matching_sps)} service providers with participant ID {injected_test.participant_id} ({matching_ids}) when exactly 1 was expected" + ) + sp = matching_sps[0] + check = self.check("Successful test deletion", [sp.participant_id]) + try: + query = sp.delete_test(injected_test.test_id, injected_test.version) + self.record_query(query) + if query.status_code != 200: + raise ValueError( + f"Received status code {query.status_code} after attempting to delete test {injected_test.test_id} at version {injected_test.version} from service provider {sp.participant_id}" + ) + check.record_passed() + except (RequestException, ValueError) as e: + stacktrace = "".join( + traceback.format_exception(type(e), value=e, tb=e.__traceback__) + ) + check.record_failed( + summary="Error while trying to delete test flight", + severity=Severity.Medium, + details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}", + ) + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index 92be3e719f..090a654bed 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -1,20 +1,19 @@ -from datetime import timedelta +import time import traceback import uuid -import time from typing import List, Optional import arrow from implicitdict import ImplicitDict from loguru import logger from requests.exceptions import RequestException -from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight -from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario +from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse +from monitoring.monitorlib.rid import RIDVersion from monitoring.monitorlib.rid_automated_testing.injection_api import ( CreateTestParameters, ) -from monitoring.monitorlib.rid import RIDVersion +from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource from monitoring.uss_qualifier.resources.netrid import ( @@ -23,19 +22,18 @@ NetRIDObserversResource, EvaluationConfigurationResource, ) +from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator from monitoring.uss_qualifier.scenarios.astm.netrid.injected_flight_collection import ( InjectedFlightCollection, ) -from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( - VirtualObserver, -) -from monitoring.uss_qualifier.scenarios.scenario import TestScenario -from monitoring.uss_qualifier.scenarios.astm.netrid import display_data_evaluator from monitoring.uss_qualifier.scenarios.astm.netrid.injection import ( InjectedFlight, InjectedTest, ) -from uas_standards.interuss.automated_testing.rid.v1.injection import ChangeTestResponse +from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( + VirtualObserver, +) +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario class NominalBehavior(GenericTestScenario): @@ -66,10 +64,8 @@ def __init__( @property def _rid_version(self) -> RIDVersion: - return ( - self._dss_pool.dss_instances[0].rid_version - if self._dss_pool - else RIDVersion.f3411_19 + raise NotImplementedError( + "NominalBehavior test scenario subclass must specify _rid_version" ) def run(self): diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 86ede08a07..c88398b7d1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -36,7 +36,11 @@ from monitoring.uss_qualifier.scenarios.astm.netrid.virtual_observer import ( VirtualObserver, ) -from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType, PendingCheck +from monitoring.uss_qualifier.scenarios.scenario import ( + TestScenarioType, + PendingCheck, + TestScenario, +) from monitoring.uss_qualifier.scenarios.astm.netrid.injection import InjectedFlight DISTANCE_TOLERANCE_M = 0.01 @@ -190,7 +194,7 @@ class RIDObservationEvaluator(object): def __init__( self, - test_scenario: TestScenarioType, + test_scenario: TestScenario, injected_flights: List[InjectedFlight], config: EvaluationConfiguration, rid_version: RIDVersion, diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py index 1552d4ee7d..ab08466a88 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py @@ -1,3 +1,4 @@ from .dss_interoperability import DSSInteroperability from .nominal_behavior import NominalBehavior +from .misbehavior import Misbehavior from .aggregate_checks import AggregateChecks diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 392c41991a..5c945cbfd7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -60,9 +60,32 @@ of the durations for the replies to requested flights in an area do not exceed t ### Verify https is in use test step -Inspects all record queries for their usage of https. If resources such as a service provide, observer or DSS are marked -as being in "local debug" mode, they may serve requests over https without breaking the test suite. +Inspects all record queries for their usage of https. If services such as a service provider, observer or DSS are marked +as being in "local debug" mode, they may serve requests over http without producing failed checks despite their lack of encryption. #### All interactions happen over https check If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. + +#### No unattributed queries check + +All queries must have been attributed to a participant: an unattributed query means that one of the test cases has not +properly recorded to which participant it was made. + +This is an internal requirement and does not necessarily imply that there is a problem with the participants under test. + +## Mock USS interactions evaluation test case + +In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify +compliance with requirements. + +### Get mock USS interactions test step + +### Evaluate mock USS interactions test step + +#### No large Display Provider queries check + +If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed +area requested, then that participant will have violated **[astm.f3411.v19.NET0240](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement this check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md index 75e04682e6..865410114f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md @@ -22,43 +22,89 @@ Perform basic operations on a single DSS instance to create subscriptions and ch ### Ensure clean workspace test step -This step ensures that we remove any subscription that may already exist for the service area. +This step ensures that we remove any subscription that may already exist for the service area. First, the DSS is queried for any applicable existing subscriptions, and then any subscriptions found are deleted. -#### Successful subscription query and cleanup check +#### Successful subscription query check -We expect to be allowed to query for existing subscriptions in order to clean them up +If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not met. -## Subscription limitations test case +#### Successful subscription deletion -### Subscription quantity limitations test step +If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not met. -The test will attempt to create 10 identical subscriptions for the same area and expect this to succeed, then create an 11th one and expect it to fail. +## Subscription quantity limitations test case + +### Create maximum number of subscriptions test step + +The test will attempt to create 10 identical subscriptions for the same area and expect this to succeed. #### Create up to the maximum allowed number of subscriptions in an area check As per **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**, the DSS API is expected to allow us to create multiple subscriptions. +### Exceed maximum number of subscriptions test step + +Now, create an 11th one and expect it to fail. + #### Enforce maximum number of subscriptions for an area check If the DSS successfully creates an 11th Subscription in the same area instead of rejecting it, it will not have performed the Subscription count validation as defined in **[astm.f3411.v19.DSS0050](../../../../../requirements/astm/f3411/v19.md)** -### Subscription duration limitations test step +### Clean up subscriptions test step -#### Enforce maximum duration of subscriptions for an area check +Clean up any subscriptions created. -**[astm.f3411.v19.DSS0060](../../../../../requirements/astm/f3411/v19.md)** any subscription to the DSS may not exceed NetDSSMaxSubscriptionDuration (24 hours). +#### Successful subscription query check + +If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not met. + +#### Successful subscription deletion check + +If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not met. + +## Subscription duration limitations test case + +### Try to create too-long subscription test step The test will attempt to create a subscription for 24 hours and 10 minutes, and expect this to fail with an HTTP 400 error. -If the creation succeeds, the test expects that the effectively created subscription has been truncated to 24 hours, with a tolerance of minus 1 minute. +#### Too-long subscription creation rejected check + +**[astm.f3411.v19.DSS0060](../../../../../requirements/astm/f3411/v19.md)** any subscription to the DSS may not exceed NetDSSMaxSubscriptionDuration (24 hours). + +If creation of the too-long-duration subscription succeeds, the test expects that the effectively created subscription has been truncated to 24 hours, with a tolerance of minus 1 minute. + +### Try to extend subscription test step -It will also attempt to create a valid subscription of 23 hours an 59 minutes, and then increase its duration to 24 hours and 10 minutes, +The test will attempt to create a valid subscription of 23 hours and 59 minutes, and then increase its duration to 24 hours and 10 minutes, expecting this update to fail. -## Cleanup +#### Valid subscription created check + +The ability to create a valid subscription is required in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription duration limited during update check + +If the DSS allows a user to extend an existing, valid subscription to a duration exceeding that specified in **[astm.f3411.v19.DSS0060](../../../../../requirements/astm/f3411/v19.md)**, this check will fail. + +### Remove subscription test step + +To clean up after itself, the test deletes the subscription created in the previous step. + +#### Subscription deleted check -### Successful subscription query and cleanup check +The ability to delete an existing subscription is required in **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)**. + +## Cleanup The cleanup phase of this test scenario will remove any subscription that may have been created during the test and that intersects with the test ISA. + +### Successful subscription query check + +If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not met. + +### Successful subscription deletion + +If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v19.DSS0030](../../../../../requirements/astm/f3411/v19.md)** was not met. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md index 461a013678..f822ff8e74 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md @@ -152,6 +152,8 @@ Qualitatively proves: ISA[*P*] modified with proper response, all Subscription[i #### ISA modification triggers subscription notification requests check +TODO: Implement + **[astm.f3411.v19.A2-6-1,3c](../../../../requirements/astm/f3411/v19.md)** ### S6 test step @@ -274,6 +276,8 @@ Qualitatively proves: Expired Subscription removed from ID index on primary DSS #### 404 with proper response check +TODO: Investigate expected behavior and "404 with proper response" check + **[astm.f3411.v19.A2-6-1,3d](../../../../requirements/astm/f3411/v19.md)** ### S15 test step diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md new file mode 100644 index 0000000000..0602647f31 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -0,0 +1,60 @@ +# ASTM NetRID SP clients misbehavior handling test scenario + +## Overview + +In this scenario, the service provider's endpoint are accessed directly with missing or incorrect credentials. Resources that exists as well as resources that are not expected to exist are queried. + +## Resources + +### flights_data + +A [`FlightDataResource`](../../../../resources/netrid/flight_data.py) containing 1 nominal flight per SP under test. + +### service_providers + +A set of [`NetRIDServiceProviders`](../../../../resources/netrid/service_providers.py) to be tested for proper request authentication. This scenario requires at least one SP under test. + +### evaluation_configuration + +This [`EvaluationConfigurationResource`](../../../../resources/netrid/evaluation.py) defines how to gauge success when observing the injected flights. + +### dss_pool + +A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required for providing the qualifier with the flights URL of the service providers being tested. + +## Unauthenticated requests test case + +### Injection test step + +In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. + +#### Successful injection check + +Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. + +**[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. +This check will fail if the flight was not successfully injected. + +#### Identifiable flights check + +This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. + +### Unauthenticated requests test step + +In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials +to confirm that the requested data is indeed available to any authorized query. + +It then repeats the exact same request while omitting the credentials, and expects this to fail. + +#### Missing credentials check + +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)**, +and that requests for existing flights that are executed with missing or incorrect credentials fail. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove injected data from all SPs. + +### Successful test deletion check + +**[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py new file mode 100644 index 0000000000..a45925f919 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.py @@ -0,0 +1,11 @@ +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid.common.misbehavior import ( + Misbehavior as CommonMisbehavior, +) + + +class Misbehavior(TestScenario, CommonMisbehavior): + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_19 diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 803088ea41..1318567e73 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -41,6 +41,8 @@ This check will fail if the flight was not successfully injected. #### Valid flight check +TODO: Validate injected flights, especially to make sure they contain the specified injection IDs + Per **[interuss.automated_testing.rid.injection.UpsertTestResult](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the NetRID Service Provider under test should only make valid modifications to the injected flights. This includes: * A flight with the specified injection ID must be returned. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py index 19713cd1bb..4ce255a732 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.py @@ -1,3 +1,4 @@ +from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.scenarios.astm.netrid.common.nominal_behavior import ( NominalBehavior as CommonNominalBehavior, @@ -5,4 +6,6 @@ class NominalBehavior(TestScenario, CommonNominalBehavior): - pass + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_19 diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py index 1552d4ee7d..ab08466a88 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py @@ -1,3 +1,4 @@ from .dss_interoperability import DSSInteroperability from .nominal_behavior import NominalBehavior +from .misbehavior import Misbehavior from .aggregate_checks import AggregateChecks diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 39a836e083..7e1f3e1600 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -60,9 +60,32 @@ of the durations for the replies to requested flights in an area do not exceed t ### Verify https is in use test step -Inspects all record queries for their usage of https. If resources such as a service provide, observer or DSS are marked -as being in "local debug" mode, they may serve requests over https without breaking the test suite. +Inspects all record queries for their usage of https. If services such as a service provider, observer or DSS are marked +as being in "local debug" mode, they may serve requests over http without producing failed checks despite their lack of encryption. + +#### No unattributed queries check + +All queries must have been attributed to a participant: an unattributed query means that one of the test cases has not +properly recorded to which participant it was made. + +This is an internal requirement and does not necessarily imply that there is a problem with the participants under test. #### All interactions happen over https check If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. + +## Mock USS interactions evaluation test case + +In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify +compliance with requirements. + +### Get mock USS interactions test step + +### Evaluate mock USS interactions test step + +#### No large Display Provider queries check + +If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed +area requested, then that participant will have violated **[astm.f3411.v22a.NET0240](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement this check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 2a033790da..f8b0b8d157 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -80,10 +80,14 @@ The API for **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -#### Successful ISA query check +### Successful ISA query check **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -#### Removed pre-existing ISA check +### Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +### Notified subscriber check + +When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md index 1c5fcfeaee..4d5881e62e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md @@ -22,43 +22,89 @@ Perform basic operations on a single DSS instance to create subscriptions and ch ### Ensure clean workspace test step -This step ensures that we remove any subscription that may already exist for the service area. +This step ensures that we remove any subscription that may already exist for the service area. First, the DSS is queried for any applicable existing subscriptions, and then any subscriptions found are deleted. -#### Successful subscription query and cleanup check +#### Successful subscription query check -We expect to be allowed to query for existing subscriptions in order to clean them up +If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not met. -## Subscription limitations test case +#### Successful subscription deletion -### Subscription quantity limitations test step +If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not met. -The test will attempt to create 10 identical subscriptions for the same area and expect this to succeed, then create an 11th one and expect it to fail. +## Subscription quantity limitations test case + +### Create maximum number of subscriptions test step + +The test will attempt to create 10 identical subscriptions for the same area and expect this to succeed. #### Create up to the maximum allowed number of subscriptions in an area check As per **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**, the DSS API is expected to allow us to create multiple subscriptions. +### Exceed maximum number of subscriptions test step + +Now, create an 11th one and expect it to fail. + #### Enforce maximum number of subscriptions for an area check If the DSS successfully creates an 11th Subscription in the same area instead of rejecting it, it will not have performed the Subscription count validation as defined in **[astm.f3411.v22a.DSS0050](../../../../../requirements/astm/f3411/v22a.md)** -### Subscription duration limitations test step +### Clean up subscriptions test step -#### Enforce maximum duration of subscriptions for an area check +Clean up any subscriptions created. -**[astm.f3411.v22a.DSS0060](../../../../../requirements/astm/f3411/v22a.md)** any subscription to the DSS may not exceed NetDSSMaxSubscriptionDuration (24 hours). +#### Successful subscription query check + +If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not met. + +#### Successful subscription deletion check + +If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not met. + +## Subscription duration limitations test case + +### Try to create too-long subscription test step The test will attempt to create a subscription for 24 hours and 10 minutes, and expect this to fail with an HTTP 400 error. -If the creation succeeds, the test expects that the effectively created subscription has been truncated to 24 hours, with a tolerance of minus 1 minute. +#### Too-long subscription creation rejected check + +**[astm.f3411.v22a.DSS0060](../../../../../requirements/astm/f3411/v22a.md)** any subscription to the DSS may not exceed NetDSSMaxSubscriptionDuration (24 hours). + +If creation of the too-long-duration subscription succeeds, the test expects that the effectively created subscription has been truncated to 24 hours, with a tolerance of minus 1 minute. + +### Try to extend subscription test step -It will also attempt to create a valid subscription of 23 hours an 59 minutes, and then increase its duration to 24 hours and 10 minutes, +The test will attempt to create a valid subscription of 23 hours and 59 minutes, and then increase its duration to 24 hours and 10 minutes, expecting this update to fail. -## Cleanup +#### Valid subscription created check + +The ability to create a valid subscription is required in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription duration limited during update check + +If the DSS allows a user to extend an existing, valid subscription to a duration exceeding that specified in **[astm.f3411.v22a.DSS0060](../../../../../requirements/astm/f3411/v22a.md)**, this check will fail. + +### Remove subscription test step + +To clean up after itself, the test deletes the subscription created in the previous step. + +#### Subscription deleted check -### Successful subscription query and cleanup check +The ability to delete an existing subscription is required in **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)**. + +## Cleanup The cleanup phase of this test scenario will remove any subscription that may have been created during the test and that intersects with the test ISA. + +### Successful subscription query check + +If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not met. + +### Successful subscription deletion + +If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** was not met. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md index 0c04986259..85898a4f18 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md @@ -151,6 +151,8 @@ Qualitatively proves: ISA[*P*] modified with proper response, all Subscription[i #### ISA modification triggers subscription notification requests check +TODO: Implement + **[astm.f3411.v22a.A2-6-1,3c](../../../../requirements/astm/f3411/v22a.md)** ### S6 test step @@ -273,6 +275,8 @@ Qualitatively proves: Expired Subscription removed from ID index on primary DSS #### 404 with proper response check +TODO: Investigate expected behavior and "404 with proper response" check + **[astm.f3411.v22a.A2-6-1,3d](../../../../requirements/astm/f3411/v22a.md)** ### S15 test step diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md new file mode 100644 index 0000000000..d94a021a73 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -0,0 +1,60 @@ +# ASTM NetRID SP clients misbehavior handling test scenario + +## Overview + +In this scenario, the service provider's endpoint are accessed directly with missing or incorrect credentials. Resources that exists as well as resources that are not expected to exist are queried. + +## Resources + +### flights_data + +A [`FlightDataResource`](../../../../resources/netrid/flight_data.py) containing 1 nominal flight per SP under test. + +### service_providers + +A set of [`NetRIDServiceProviders`](../../../../resources/netrid/service_providers.py) to be tested for proper request authentication. This scenario requires at least one SP under test. + +### evaluation_configuration + +This [`EvaluationConfigurationResource`](../../../../resources/netrid/evaluation.py) defines how to gauge success when observing the injected flights. + +### dss_pool + +A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required for providing the qualifier with the flights URL of the service providers being tested. + +## Unauthenticated requests test case + +### Injection test step + +In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. + +#### Successful injection check + +Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. + +**[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. +This check will fail if the flight was not successfully injected. + +#### Identifiable flights check + +This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. + +### Unauthenticated requests test step + +In order to properly test whether the SP handles authentication correctly, this step will first attempt to do a request with the proper credentials +to confirm that the requested data is indeed available to any authorized query. + +It then repeats the exact same request while omitting the credentials, and expects this to fail. + +#### Missing credentials check + +This check ensures that all requests are properly authenticated, as required by **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)**, +and that requests for existing flights that are executed with missing or incorrect credentials fail. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove injected data from all SPs. + +### Successful test deletion check + +**[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py new file mode 100644 index 0000000000..c687ea7caa --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.py @@ -0,0 +1,11 @@ +from monitoring.monitorlib.rid import RIDVersion +from monitoring.uss_qualifier.scenarios.scenario import TestScenario +from monitoring.uss_qualifier.scenarios.astm.netrid.common.misbehavior import ( + Misbehavior as CommonMisbehavior, +) + + +class Misbehavior(TestScenario, CommonMisbehavior): + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_22a diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index b5994a24eb..beb41a6b49 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -41,6 +41,8 @@ This check will fail if the flight was not successfully injected. #### Valid flight check +TODO: Validate injected flights, especially to make sure they contain the specified injection IDs + Per **[interuss.automated_testing.rid.injection.UpsertTestResult](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the NetRID Service Provider under test should only make valid modifications to the injected flights. This includes: * A flight with the specified injection ID must be returned. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py index 19713cd1bb..c0110898e4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.py @@ -1,3 +1,4 @@ +from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.scenarios.astm.netrid.common.nominal_behavior import ( NominalBehavior as CommonNominalBehavior, @@ -5,4 +6,6 @@ class NominalBehavior(TestScenario, CommonNominalBehavior): - pass + @property + def _rid_version(self) -> RIDVersion: + return RIDVersion.f3411_22a diff --git a/monitoring/uss_qualifier/scenarios/definitions.py b/monitoring/uss_qualifier/scenarios/definitions.py index 5672c40b10..90d8c556fc 100644 --- a/monitoring/uss_qualifier/scenarios/definitions.py +++ b/monitoring/uss_qualifier/scenarios/definitions.py @@ -5,9 +5,13 @@ from monitoring.uss_qualifier.resources.definitions import ResourceID +TestScenarioTypeName = str +"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the `uss_qualifier` module""" + + class TestScenarioDeclaration(ImplicitDict): - scenario_type: FileReference - """Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder""" + scenario_type: TestScenarioTypeName + """Type of test scenario.""" resources: Optional[Dict[ResourceID, ResourceID]] """Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite. diff --git a/monitoring/uss_qualifier/scenarios/documentation/definitions.py b/monitoring/uss_qualifier/scenarios/documentation/definitions.py index 1cde8ac14d..ef69179a87 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/definitions.py +++ b/monitoring/uss_qualifier/scenarios/documentation/definitions.py @@ -8,6 +8,7 @@ class TestCheckDocumentation(ImplicitDict): name: str url: Optional[str] = None applicable_requirements: List[RequirementID] + has_todo: bool class TestStepDocumentation(ImplicitDict): diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index cdec8572ba..a90227a53c 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -10,6 +10,8 @@ from monitoring import uss_qualifier as uss_qualifier_module from monitoring.monitorlib import versioning from monitoring.monitorlib.inspection import fullname, get_module_object_by_name +from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestStepDocumentation, TestCheckDocumentation, @@ -38,6 +40,8 @@ def _text_of(value) -> str: result += _text_of(child) return result elif isinstance(value, marko.inline.InlineElement): + if isinstance(value, marko.inline.LineBreak): + return "\n" if isinstance(value.children, str): return value.children result = "" @@ -73,17 +77,22 @@ def _parse_test_check( ) -> TestCheckDocumentation: name = _text_of(values[0])[0 : -len(TEST_CHECK_SUFFIX)] url = _url_of(doc_filename + anchors[values[0]]) + has_todo = False reqs: List[str] = [] c = 1 while c < len(values): if isinstance(values[c], marko.block.Paragraph): + if "TODO:" in _text_of(values[c]): + has_todo = True for child in values[c].children: if isinstance(child, marko.inline.StrongEmphasis): - reqs.append(_text_of(child)) + reqs.append(RequirementID(_text_of(child))) c += 1 - return TestCheckDocumentation(name=name, url=url, applicable_requirements=reqs) + return TestCheckDocumentation( + name=name, url=url, applicable_requirements=reqs, has_todo=has_todo + ) def _get_linked_test_step( @@ -310,6 +319,8 @@ def get_documentation(scenario: Type) -> TestScenarioDocumentation: return getattr(scenario, DOC_CACHE_ATTRIBUTE) -def get_documentation_by_name(scenario_type_name: str) -> TestScenarioDocumentation: +def get_documentation_by_name( + scenario_type_name: TestScenarioTypeName, +) -> TestScenarioDocumentation: scenario_type = get_module_object_by_name(uss_qualifier_module, scenario_type_name) return get_documentation(scenario_type) diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py index 6024aa62b1..81f5fb4928 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py @@ -92,6 +92,7 @@ def _plan_flights(self): name=c.name, url=c.url, applicable_requirements=row.requirement_ids, + has_todo=False, ) for c in checks ] diff --git a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py index efb0adc6ee..1ba899cd98 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py +++ b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py @@ -63,6 +63,7 @@ def _map_query(self): name=check_name, url=check_url, applicable_requirements=row.requirement_ids, + has_todo=False, ) doc = TestStepDocumentation( name=row.geospatial_check_id, diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index b3901e938e..a411543358 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -24,7 +24,10 @@ ParticipantID, PassedCheck, ) -from monitoring.uss_qualifier.scenarios.definitions import TestScenarioDeclaration +from monitoring.uss_qualifier.scenarios.definitions import ( + TestScenarioDeclaration, + TestScenarioTypeName, +) from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestScenarioDocumentation, TestCaseDocumentation, @@ -159,7 +162,7 @@ def record_passed( self._step_report.passed_checks.append(passed_check) -def get_scenario_type_by_name(scenario_type_name: str) -> Type: +def get_scenario_type_by_name(scenario_type_name: TestScenarioTypeName) -> Type: inspection.import_submodules(scenarios_module) scenario_type = inspection.get_module_object_by_name( parent_module=uss_qualifier_module, object_name=scenario_type_name @@ -361,11 +364,19 @@ def check( check_list = ", ".join(available_checks) if self._allow_undocumented_checks: check_documentation = TestCheckDocumentation( - name=name, applicable_requirements=[] + name=name, applicable_requirements=[], has_todo=False ) else: + test_step_name = ( + self._current_step.name if self._current_step else "" + ) + test_case_name = ( + self._current_case.name + if self._current_case + else "" + ) raise RuntimeError( - f'Test scenario `{self.me()}` was instructed to prepare to record outcome for check "{name}" during test step "{self._current_step.name}" during test case "{self._current_case.name}", but that check is not declared in documentation; declared checks are: {check_list}' + f'Test scenario `{self.me()}` was instructed to prepare to record outcome for check "{name}" during test step "{test_step_name}" during test case "{test_case_name}", but that check is not declared in documentation; declared checks are: {check_list}' ) return PendingCheck( phase=self._phase, diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index b91c6ea824..089ea93df4 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -4,6 +4,268 @@ ## Actions -1. Action generator: `action_generators.astm.f3411.ForEachDSS` +1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) + 1. Suite: [DSS instance probing for ASTM NetRID F3411-19](f3411_19/dss_probing.md) ([`suites.astm.netrid.f3411_19.dss_probing`](f3411_19/dss_probing.yaml)) 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) +4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) +4. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3411
.v19		A2-6-1,1aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3cIn progressASTM F3411-19 NetRID DSS interoperability
A2-6-1,3dIn progressASTM F3411-19 NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-19 NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130ImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-19 NetRID DSS interoperability
DSS0210ImplementedASTM F3411-19 NetRID DSS interoperability
NET0240PlannedASTM F3411-19 NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-19 NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-19 NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-19 NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID nominal behavior
interuss
.automated_testing
.rid
.injection		DeleteTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultPlannedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
interuss
.automated_testing
.rid
.observation		ObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml index cd07701e54..0d41a535fd 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml @@ -38,6 +38,15 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.Misbehavior + resources: + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + dss_pool: dss_instances + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v19.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index 5b9b915276..2bf986cf77 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -5,3 +5,30 @@ ## Actions 1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3411
.v19		DSS0030ImplementedASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md new file mode 100644 index 0000000000..2fe606005f --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md @@ -0,0 +1,34 @@ + +# DSS instance validator test suite +Defined in [parent suite](f3411_19.md) [`suites.astm.netrid.f3411_19`](./f3411_19.yaml) + +## Actions + +1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aDSS0030ImplementedASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 8dd20db5b5..6096cf8563 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -4,6 +4,313 @@ ## Actions -1. Action generator: `action_generators.astm.f3411.ForEachDSS` +1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) + 1. Suite: [DSS instance probing for ASTM NetRID F3411-22a](f3411_22a/dss_probing.md) ([`suites.astm.netrid.f3411_22a.dss_probing`](f3411_22a/dss_probing.yaml)) 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) +4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v22a/misbehavior.md) ([`scenarios.astm.netrid.v22a.Misbehavior`](../../../scenarios/astm/netrid/v22a/misbehavior.py)) +4. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3411
.v22a		A2-6-1,1aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cIn progressASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dIn progressASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0210ImplementedASTM F3411-22a NetRID DSS interoperability
NET0240PlannedASTM F3411-22a NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260,Table1,1ImplementedASTM NetRID nominal behavior
NET0260,Table1,1aImplementedASTM NetRID nominal behavior
NET0260,Table1,23ImplementedASTM NetRID nominal behavior
NET0260,Table1,24ImplementedASTM NetRID nominal behavior
NET0260,Table1,25ImplementedASTM NetRID nominal behavior
NET0260,Table1,26ImplementedASTM NetRID nominal behavior
NET0260,Table1,7ImplementedASTM NetRID nominal behavior
NET0260,Table1,9ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-22a NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-22a NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-22a NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
NET0730ImplementedASTM NetRID DSS: Simple ISA
interuss
.automated_testing
.rid
.injection		DeleteTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultPlannedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
interuss
.automated_testing
.rid
.observation		ObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml index 528283391f..1269c75fa3 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml @@ -40,6 +40,15 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v22a.Misbehavior + resources: + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + dss_pool: dss_instances + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v22a.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md index dc68ba0f78..8a60f30246 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md @@ -6,3 +6,40 @@ 1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) 2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3411
.v22a		DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
NET0710ImplementedASTM NetRID DSS: Simple ISA
NET0730ImplementedASTM NetRID DSS: Simple ISA
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md new file mode 100644 index 0000000000..0c90a2bfb6 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md @@ -0,0 +1,40 @@ + +# DSS instance validator test suite +Defined in [parent suite](f3411_22a.md) [`suites.astm.netrid.f3411_22a`](./f3411_22a.yaml) + +## Actions + +1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) +2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aDSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
NET0710ImplementedASTM NetRID DSS: Simple ISA
diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 62a6f5c093..b6f70a1d08 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -4,6 +4,127 @@ ## Actions -1. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` -2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` -3. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +1. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) +2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) +3. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3548
.v21		DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss
.automated_testing
.flight_planning		ClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index ca453656d1..8ad4f3c6e6 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -1,18 +1,59 @@ +from __future__ import annotations import glob import inspect import os -from typing import Iterator, Optional +from dataclasses import dataclass +from typing import Iterator, Optional, List, Union, Dict from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, +) +from monitoring.uss_qualifier.action_generators.definitions import ( + ActionGeneratorDefinition, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialActionGeneratorAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_generator_definition, +) from monitoring.uss_qualifier.fileio import ( load_dict_with_references, get_package_name, resolve_filename, + FileReference, +) +from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.requirements.documentation import get_requirement +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName +from monitoring.uss_qualifier.scenarios.documentation.definitions import ( + TestScenarioDocumentation, + TestCheckDocumentation, +) +from monitoring.uss_qualifier.scenarios.documentation.parsing import ( + get_documentation, + get_documentation_by_name, ) -from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name -from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition, ActionType +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteDefinition, + ActionType, + TestSuiteActionDeclaration, +) +from monitoring.uss_qualifier.suites.suite import TestSuiteAction + + +@dataclass +class TestSuiteRenderContext(object): + parent_yaml_file: str + parent_doc_file: str + base_path: str + list_index: int + indent: int + test_suites: Dict[str, str] def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: @@ -28,63 +69,361 @@ def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: yield suite -def make_test_suite_documentation(test_suite_yaml_file: str) -> str: +def make_test_suite_documentation( + suite_def: TestSuiteDefinition, + suite_yaml_file: str, + suite_doc_file: str, + parent_suite_doc: Optional[str] = None, +) -> Dict[str, str]: + test_suites: Dict[str, str] = {} + lines = [] - suite_def: TestSuiteDefinition = ImplicitDict.parse( - load_dict_with_references("file://" + test_suite_yaml_file), TestSuiteDefinition - ) lines.append( "" ) lines.append(f"# {suite_def.name} test suite") - local_path = os.path.split(test_suite_yaml_file)[-1] - lines.append(f"[`{get_package_name(test_suite_yaml_file)}`](./{local_path})") + local_path = os.path.split(suite_yaml_file)[-1] + if parent_suite_doc is None: + prefix = "" + else: + parent_rel_path = os.path.relpath( + parent_suite_doc, start=os.path.dirname(suite_doc_file) + ) + prefix = f"Defined in [parent suite]({parent_rel_path}) " + lines.append(f"{prefix}[`{get_package_name(suite_yaml_file)}`](./{local_path})") lines.append("") lines.append("## Actions") lines.append("") - base_path = os.path.dirname(test_suite_yaml_file) + base_path = os.path.dirname(suite_yaml_file) + i = 0 for i, action in enumerate(suite_def.actions): - action_type = action.get_action_type() - if action_type == ActionType.TestScenario: - scenario_type = get_scenario_type_by_name( - action.test_scenario.scenario_type + lines.extend( + _render_action( + action, + TestSuiteRenderContext( + parent_yaml_file=suite_yaml_file, + parent_doc_file=suite_doc_file, + base_path=base_path, + list_index=i + 1, + indent=0, + test_suites=test_suites, + ), ) - py_rel_path = os.path.relpath(inspect.getfile(scenario_type), base_path) - scenario_doc = get_documentation(scenario_type) - doc_rel_path = os.path.relpath(scenario_doc.local_path, start=base_path) - lines.append( - f"{i + 1}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{action.test_scenario.scenario_type}`]({py_rel_path}))" + ) + if ( + "report_evaluation_scenario" in suite_def + and suite_def.report_evaluation_scenario + ): + lines.extend( + _render_scenario( + suite_def.report_evaluation_scenario.scenario_type, + TestSuiteRenderContext( + parent_yaml_file=suite_yaml_file, + parent_doc_file=suite_doc_file, + base_path=base_path, + list_index=i + 1, + indent=0, + test_suites=test_suites, + ), ) - elif action_type == ActionType.TestSuite: - if "suite_type" in action.test_suite and action.test_suite.suite_type: - suite_def = ImplicitDict.parse( - load_dict_with_references(action.test_suite.suite_type), - TestSuiteDefinition, - ) - suite_path = resolve_filename(action.test_suite.suite_type) - suite_rel_path = os.path.relpath(suite_path, start=base_path) - doc_path = os.path.splitext(suite_path)[0] + ".md" - doc_rel_path = os.path.relpath(doc_path, start=base_path) - lines.append( - f"{i + 1}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{action.test_suite.suite_type}`]({suite_rel_path}))" + ) + lines.append("") + + lines.append("## Checked requirements") + lines.append("") + reqs = _collect_requirements_from_suite_def(suite_def) + if not reqs: + lines.append( + "_This test suite documentation does not indicate that any requirements are checked._" + ) + else: + # Use an HTML table rather than Markdown table to enabled advanced features like spans + lines.append("") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + + req_ids_by_package: Dict[str, List[RequirementID]] = {} + for req_id, req in reqs.items(): + package = req_ids_by_package.get(req_id.package(), []) + if req_id not in package: + package.append(req_id) + req_ids_by_package[req_id.package()] = package + + for package in sorted(req_ids_by_package): + req_md_path = os.path.relpath( + req_ids_by_package[package][0].md_file_path(), start=base_path + ) + package_caption = "
.".join(package.split(".")) + package_line = f' ' + for req_id in sorted(req_ids_by_package[package]): + req_text = f'{req_id.requirement_name()}' + + has_todo = False + has_complete = False + scenarios = {} + for checked_in in reqs[req_id].checked_in: + if checked_in.scenario.local_path not in scenarios: + scenarios[checked_in.scenario.name] = checked_in.scenario + if checked_in.check.has_todo: + has_todo = True + else: + has_complete = True + if has_complete and not has_todo: + status_text = "Implemented" + elif has_todo and not has_complete: + status_text = "Planned" + elif has_todo and has_complete: + status_text = "In progress" + else: + status_text = "Not implemented" + checked_in = list( + f'{scenarios[s].name}' + for s in sorted(scenarios) ) - elif "suite_definition" in action.test_suite and action.suite_definition: - # TODO: Generate additional test suite documentation for in-suite suite definition - lines.append(f"{i + 1}. Suite: ") + checked_in_text = f"{'
'.join(checked_in)}" + + lines.append(" ") + if package_line: + lines.append(package_line) + package_line = None + lines.append(f" ") + lines.append(f" ") + lines.append(f" ") + lines.append(" ") + lines.append("
PackageRequirementStatusChecked in
{package_caption}
{req_text}{status_text}{checked_in_text}
") + lines.append("") + + test_suites[suite_doc_file] = "\n".join(lines) + return test_suites + + +def _render_scenario( + scenario_type_name: TestScenarioTypeName, context: TestSuiteRenderContext +) -> List[str]: + lines = [] + scenario_type = get_scenario_type_by_name(scenario_type_name) + py_rel_path = os.path.relpath(inspect.getfile(scenario_type), context.base_path) + scenario_doc = get_documentation(scenario_type) + doc_rel_path = os.path.relpath(scenario_doc.local_path, start=context.base_path) + lines.append( + f"{' ' * context.indent}{context.list_index}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{scenario_type_name}`]({py_rel_path}))" + ) + return lines + + +def _render_suite_by_type( + suite_type: FileReference, context: TestSuiteRenderContext +) -> List[str]: + lines = [] + suite_def = ImplicitDict.parse( + load_dict_with_references(suite_type), + TestSuiteDefinition, + ) + suite_path = resolve_filename(suite_type) + suite_rel_path = os.path.relpath(suite_path, start=context.base_path) + doc_path = os.path.splitext(suite_path)[0] + ".md" + doc_rel_path = os.path.relpath(doc_path, start=context.base_path) + lines.append( + f"{' ' * context.indent}{context.list_index}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{suite_type}`]({suite_rel_path}))" + ) + return lines + + +def _render_suite_by_definition( + suite_def: TestSuiteDefinition, context: TestSuiteRenderContext +) -> List[str]: + doc_path = ( + os.path.splitext(context.parent_doc_file)[0] + f"_suite{context.list_index}.md" + ) + new_docs = make_test_suite_documentation( + suite_def, context.parent_yaml_file, doc_path, context.parent_doc_file + ) + + for k, v in new_docs.items(): + context.test_suites[k] = v + + doc_rel_path = os.path.relpath(doc_path, context.base_path) + parent_rel_path = os.path.relpath(context.parent_yaml_file, start=context.base_path) + return [ + f"{' ' * context.indent}{context.list_index}. Suite: [{suite_def.name}]({doc_rel_path}) ([in-suite definition]({parent_rel_path}))" + ] + + +def _render_action_generator( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction], + context: TestSuiteRenderContext, +) -> List[str]: + lines = [] + action_generator_type = action_generator_type_from_name( + generator_def.generator_type + ) + py_rel_path = os.path.relpath( + inspect.getfile(action_generator_type), start=context.base_path + ) + lines.append( + f"{' ' * context.indent}{context.list_index}. Action generator: [`{generator_def.generator_type}`]({py_rel_path})" + ) + potential_actions = list_potential_actions_for_action_generator_definition( + generator_def + ) + for j, potential_action in enumerate(potential_actions): + lines.extend( + _render_action( + potential_action, + TestSuiteRenderContext( + parent_yaml_file=context.parent_yaml_file, + parent_doc_file=context.parent_doc_file, + base_path=context.base_path, + list_index=j + 1, + indent=context.indent + 4, + test_suites=context.test_suites, + ), + ) + ) + return lines + + +def _render_action( + action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction], + context: TestSuiteRenderContext, +) -> List[str]: + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + return _render_scenario(action.test_scenario.scenario_type, context) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + return _render_suite_by_type(action.test_suite.suite_type, context) + elif ( + "suite_definition" in action.test_suite + and action.test_suite.suite_definition + ): + return _render_suite_by_definition( + action.test_suite.suite_definition, context + ) + else: + raise ValueError( + f"Test suite action {context.list_index} missing suite type or definition" + ) + elif action_type == ActionType.ActionGenerator: + return _render_action_generator(action.action_generator, context) + else: + raise NotImplementedError(f"Unsupported test suite action type: {action_type}") + + +@dataclass +class SuiteLocation(object): + scenario: TestScenarioDocumentation + check: TestCheckDocumentation + + +@dataclass +class RequirementInSuite(object): + checked_in: List[SuiteLocation] + + def extend(self, other: RequirementInSuite): + self.checked_in.extend(other.checked_in) + + +def _collect_requirements_from_suite_def( + suite_def: TestSuiteDefinition, +) -> Dict[RequirementID, RequirementInSuite]: + reqs: Dict[RequirementID, RequirementInSuite] = {} + + def combine(new_reqs: Dict[RequirementID, RequirementInSuite]) -> None: + for req_id, req in new_reqs.items(): + if req_id not in reqs: + reqs[req_id] = req else: - raise ValueError( - f"Test suite action {i + 1} missing suite type or definition in {test_suite_yaml_file}" - ) - elif action_type == ActionType.ActionGenerator: - # TODO: Add documentation for action generators - lines.append( - f"{i + 1}. Action generator: `{action.action_generator.generator_type}`" + reqs[req_id].extend(req) + + for action in suite_def.actions: + combine(_collect_requirements_from_action(action)) + if ( + "report_evaluation_scenario" in suite_def + and suite_def.report_evaluation_scenario + ): + combine( + _collect_requirements_from_scenario( + suite_def.report_evaluation_scenario.scenario_type + ) + ) + return reqs + + +def _collect_requirements_from_action( + action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction] +) -> Dict[RequirementID, RequirementInSuite]: + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + return _collect_requirements_from_scenario(action.test_scenario.scenario_type) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + suite_def = ImplicitDict.parse( + load_dict_with_references(action.test_suite.suite_type), + TestSuiteDefinition, + ) + return _collect_requirements_from_suite_def(suite_def) + elif ( + "suite_definition" in action.test_suite + and action.test_suite.suite_definition + ): + return _collect_requirements_from_suite_def( + action.test_suite.suite_definition ) else: - raise NotImplementedError( - f"Unsupported test suite action type: {action_type}" + raise ValueError( + "Neither suite_type nor suite_definition specified in test_suite action" ) + elif action_type == ActionType.ActionGenerator: + return _collect_requirements_from_action_generator(action.action_generator) + else: + raise NotImplementedError( + f"Test suite action type {action_type} not yet supported" + ) - lines.append("") - return "\n".join(lines) + +def _collect_requirements_from_scenario( + scenario_type: TestScenarioTypeName, +) -> Dict[RequirementID, RequirementInSuite]: + docs = get_documentation_by_name(scenario_type) + reqs: Dict[RequirementID, RequirementInSuite] = {} + + def add_req(req_id: RequirementID, check: TestCheckDocumentation) -> None: + req = reqs.get(req_id, RequirementInSuite(checked_in=[])) + req.checked_in.append(SuiteLocation(scenario=docs, check=check)) + reqs[req_id] = req + + for case in docs.cases: + for step in case.steps: + for check in step.checks: + for req_id in check.applicable_requirements: + add_req(req_id, check) + if "cleanup" in docs and docs.cleanup: + for check in docs.cleanup.checks: + for req_id in check.applicable_requirements: + add_req(req_id, check) + return reqs + + +def _collect_requirements_from_action_generator( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction] +) -> Dict[RequirementID, RequirementInSuite]: + potential_actions = list_potential_actions_for_action_generator_definition( + generator_def + ) + + reqs: Dict[RequirementID, RequirementInSuite] = {} + for potential_action in potential_actions: + new_reqs = _collect_requirements_from_action(potential_action) + for req_id, req in new_reqs.items(): + if req_id not in reqs: + reqs[req_id] = req + else: + reqs[req_id].extend(req) + + return reqs diff --git a/monitoring/uss_qualifier/suites/documentation/format_documentation.py b/monitoring/uss_qualifier/suites/documentation/format_documentation.py index 10b3ea2a5a..680ef89e3d 100644 --- a/monitoring/uss_qualifier/suites/documentation/format_documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/format_documentation.py @@ -2,6 +2,11 @@ import os import sys +from implicitdict import ImplicitDict +from monitoring.monitorlib.inspection import import_submodules +from monitoring.uss_qualifier import scenarios, suites, action_generators +from monitoring.uss_qualifier.fileio import load_dict_with_references +from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition from monitoring.uss_qualifier.suites.documentation.documentation import ( find_test_suites, make_test_suite_documentation, @@ -9,10 +14,24 @@ def main(lint: bool) -> int: - changes = False + import_submodules(scenarios) + import_submodules(suites) + import_submodules(action_generators) + + test_suite_docs = {} for suite_yaml_file in find_test_suites(): - suite_doc_content = make_test_suite_documentation(suite_yaml_file) + suite_def: TestSuiteDefinition = ImplicitDict.parse( + load_dict_with_references("file://" + suite_yaml_file), TestSuiteDefinition + ) suite_doc_file = os.path.splitext(suite_yaml_file)[0] + ".md" + new_docs = make_test_suite_documentation( + suite_def, suite_yaml_file, suite_doc_file + ) + for k, v in new_docs.items(): + test_suite_docs[k] = v + + changes = False + for suite_doc_file, suite_doc_content in test_suite_docs.items(): if os.path.exists(suite_doc_file): with open(suite_doc_file, "r") as f: existing_content = f.read() @@ -21,7 +40,9 @@ def main(lint: bool) -> int: continue changes = True if lint: - print(f"Test suite documentation must be regenerated: {suite_doc_file}") + print( + f"Test suite documentation must be regenerated with `make format`: {suite_doc_file}" + ) else: with open(suite_doc_file, "w") as f: f.write(suite_doc_content) diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index a5fe6c401d..3cfd3ac839 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -7,3 +7,121 @@ 1. Scenario: [Start message signing](../../../scenarios/faa/uft/message_signing_start.md) ([`scenarios.faa.uft.StartMessageSigningReport`](../../../scenarios/faa/uft/message_signing_start.py)) 2. Suite: [ASTM F3548-21](../../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../../astm/utm/f3548_21.yaml)) 3. Scenario: [Finalize message signing](../../../scenarios/faa/uft/message_signing_finalize.md) ([`scenarios.faa.uft.FinalizeMessageSigningReport`](../../../scenarios/faa/uft/message_signing_finalize.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3548
.v21		DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss
.automated_testing
.flight_planning		ClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 25bdb21bce..22af2d494b 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -5,4 +5,123 @@ ## Actions 1. Suite: [ASTM F3548-21](../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../astm/utm/f3548_21.yaml)) -2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Flight authorisation validation](../../scenarios/uspace/flight_auth/validation.md) ([`scenarios.uspace.flight_auth.Validation`](../../scenarios/uspace/flight_auth/validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3548
.v21		DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss
.automated_testing
.flight_planning		ClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
diff --git a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md index 910b3700fb..22d7140529 100644 --- a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md +++ b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md @@ -5,3 +5,7 @@ ## Actions 1. Scenario: [EUROCAE ED-269 UAS geographical zone model](../../scenarios/eurocae/ed269/source_data_model.md) ([`scenarios.eurocae.ed269.source_data_model.SourceDataModelValidation`](../../scenarios/eurocae/ed269/source_data_model.py)) + +## Checked requirements + +_This test suite documentation does not indicate that any requirements are checked._ diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 1cfec80a32..ec7130edb8 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -5,3 +5,307 @@ ## Actions 1. Suite: [ASTM F3411-22a](../astm/netrid/f3411_22a.md) ([`suites.astm.netrid.f3411_22a`](../astm/netrid/f3411_22a.yaml)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3411
.v22a		A2-6-1,1aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cIn progressASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dIn progressASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0210ImplementedASTM F3411-22a NetRID DSS interoperability
NET0240PlannedASTM F3411-22a NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260,Table1,1ImplementedASTM NetRID nominal behavior
NET0260,Table1,1aImplementedASTM NetRID nominal behavior
NET0260,Table1,23ImplementedASTM NetRID nominal behavior
NET0260,Table1,24ImplementedASTM NetRID nominal behavior
NET0260,Table1,25ImplementedASTM NetRID nominal behavior
NET0260,Table1,26ImplementedASTM NetRID nominal behavior
NET0260,Table1,7ImplementedASTM NetRID nominal behavior
NET0260,Table1,9ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-22a NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-22a NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-22a NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
NET0730ImplementedASTM NetRID DSS: Simple ISA
interuss
.automated_testing
.rid
.injection		DeleteTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultPlannedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
interuss
.automated_testing
.rid
.observation		ObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 6598ea51be..1e24a9d65f 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -6,3 +6,414 @@ 1. Suite: [U-space flight authorisation](flight_auth.md) ([`suites.uspace.flight_auth`](flight_auth.yaml)) 2. Suite: [U-Space network identification](network_identification.md) ([`suites.uspace.network_identification`](network_identification.yaml)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm
.f3411
.v22a		A2-6-1,1aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cIn progressASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dIn progressASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0210ImplementedASTM F3411-22a NetRID DSS interoperability
NET0240PlannedASTM F3411-22a NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260,Table1,1ImplementedASTM NetRID nominal behavior
NET0260,Table1,1aImplementedASTM NetRID nominal behavior
NET0260,Table1,23ImplementedASTM NetRID nominal behavior
NET0260,Table1,24ImplementedASTM NetRID nominal behavior
NET0260,Table1,25ImplementedASTM NetRID nominal behavior
NET0260,Table1,26ImplementedASTM NetRID nominal behavior
NET0260,Table1,7ImplementedASTM NetRID nominal behavior
NET0260,Table1,9ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-22a NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-22a NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-22a NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
NET0730ImplementedASTM NetRID DSS: Simple ISA
astm
.f3548
.v21		DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss
.automated_testing
.flight_planning		ClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss
.automated_testing
.rid
.injection		DeleteTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultPlannedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID SP clients misbehavior handling
ASTM NetRID nominal behavior
interuss
.automated_testing
.rid
.observation		ObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json b/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json index 60d2424977..26e32cbfaf 100644 --- a/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json +++ b/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json @@ -1,35 +1,35 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "resources": { - "type": [ - "object", - "null" - ], + "additionalProperties": { + "type": "string" + }, + "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource .", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" } }, - "additionalProperties": { - "type": "string" - }, - "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource ." + "type": [ + "object", + "null" + ] }, "scenario_type": { - "type": "string", - "description": "Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder" + "description": "Type of test scenario.", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", - "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "required": [ "scenario_type" - ] + ], + "type": "object" } \ No newline at end of file