diff --git a/monitoring/uss_qualifier/action_generators/action_generator.py b/monitoring/uss_qualifier/action_generators/action_generator.py index 71694d5547..5505dd8d04 100644 --- a/monitoring/uss_qualifier/action_generators/action_generator.py +++ b/monitoring/uss_qualifier/action_generators/action_generator.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod import inspect -from typing import Generic, Dict, Optional, TypeVar +from typing import Generic, Dict, Optional, TypeVar, List, Type from implicitdict import ImplicitDict from monitoring import uss_qualifier as uss_qualifier_module @@ -12,6 +12,10 @@ from monitoring.uss_qualifier.action_generators.definitions import ( ActionGeneratorSpecificationType, ActionGeneratorDefinition, + GeneratorTypeName, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, ) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID @@ -45,32 +49,35 @@ def run_next_action(self) -> Optional[TestSuiteActionReport]: "A concrete action generator must implement `actions` method" ) + @classmethod + def list_potential_actions( + cls, specification: Optional[ActionGeneratorSpecificationType] + ) -> List[PotentialGeneratedAction]: + """Enumerate the potential actions that may be run by an instance of this generator type. + + Concrete subclasses of ActionGenerator must implement a classmethod that shadows this one according to this + specification. + + Args: + specification: A serializable (subclass of implicitdict.ImplicitDict) specification for how to create the + action generator instance, or None if the action generator type does not need a specification. + + Returns: All potential actions that may be generated by this generator, depending on the resources provided. + """ + raise NotImplementedError( + "A concrete action generator must implement `list_potential_actions` classmethod" + ) + @staticmethod def make_from_definition( definition: ActionGeneratorDefinition, resources: Dict[ResourceID, ResourceType] ) -> ActionGeneratorType: - from monitoring.uss_qualifier import ( - action_generators as action_generators_module, + action_generator_type = action_generator_type_from_name( + definition.generator_type ) + specification_type = action_generator_specification_type(action_generator_type) - import_submodules(action_generators_module) - action_generator_type = get_module_object_by_name( - parent_module=uss_qualifier_module, - object_name=definition.generator_type, - ) - if not issubclass(action_generator_type, ActionGenerator): - raise NotImplementedError( - "Action generator type {} is not a subclass of the ActionGenerator base class".format( - action_generator_type.__name__ - ) - ) - constructor_signature = inspect.signature(action_generator_type.__init__) - specification_type = None constructor_args = {} - for arg_name, arg in constructor_signature.parameters.items(): - if arg_name == "specification": - specification_type = arg.annotation - break if specification_type is not None: constructor_args["specification"] = ImplicitDict.parse( definition.specification, specification_type @@ -82,3 +89,36 @@ def make_from_definition( ActionGeneratorType = TypeVar("ActionGeneratorType", bound=ActionGenerator) + + +def action_generator_type_from_name( + action_generator_type_name: GeneratorTypeName, +) -> Type[ActionGenerator]: + from monitoring.uss_qualifier import ( + action_generators as action_generators_module, + ) + + import_submodules(action_generators_module) + action_generator_type = get_module_object_by_name( + parent_module=uss_qualifier_module, + object_name=action_generator_type_name, + ) + if not issubclass(action_generator_type, ActionGenerator): + raise NotImplementedError( + "Action generator type {} is not a subclass of the ActionGenerator base class".format( + action_generator_type.__name__ + ) + ) + return action_generator_type + + +def action_generator_specification_type( + action_generator_type: Type[ActionGenerator], +) -> Optional[Type]: + constructor_signature = inspect.signature(action_generator_type.__init__) + specification_type = None + for arg_name, arg in constructor_signature.parameters.items(): + if arg_name == "specification": + specification_type = arg.annotation + break + return specification_type diff --git a/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py b/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py index 95c756d4c0..bbbee1a469 100644 --- a/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py +++ b/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py @@ -3,6 +3,12 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.astm.f3411 import ( DSSInstanceResource, @@ -34,6 +40,14 @@ class ForEachDSS(ActionGenerator[ForEachDSSSpecification]): _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: ForEachDSSSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: ForEachDSSSpecification, diff --git a/monitoring/uss_qualifier/action_generators/definitions.py b/monitoring/uss_qualifier/action_generators/definitions.py index 8533a5ccc1..aca51c8bfa 100644 --- a/monitoring/uss_qualifier/action_generators/definitions.py +++ b/monitoring/uss_qualifier/action_generators/definitions.py @@ -5,7 +5,7 @@ GeneratorTypeName = str -"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier.action_generators` module""" +"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier` module""" ActionGeneratorSpecificationType = TypeVar( diff --git a/monitoring/uss_qualifier/action_generators/documentation/__init__.py b/monitoring/uss_qualifier/action_generators/documentation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/action_generators/documentation/definitions.py b/monitoring/uss_qualifier/action_generators/documentation/definitions.py new file mode 100644 index 0000000000..33bb951e80 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/documentation/definitions.py @@ -0,0 +1,40 @@ +from typing import Optional + +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName +from monitoring.uss_qualifier.fileio import FileReference +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName +from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition + + +class PotentialTestScenarioAction(ImplicitDict): + scenario_type: TestScenarioTypeName + """Type of test scenario.""" + + +class PotentialTestSuiteAction(ImplicitDict): + suite_type: Optional[FileReference] + """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" + + suite_definition: Optional[TestSuiteDefinition] + """Definition of test suite internal to the configuration -- specified instead of `suite_type`.""" + + +class PotentialActionGeneratorAction(ImplicitDict): + generator_type: GeneratorTypeName + """Type of action generator.""" + + specification: dict + """Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`""" + + +class PotentialGeneratedAction(ImplicitDict): + test_scenario: Optional[PotentialTestScenarioAction] + test_suite: Optional[PotentialTestSuiteAction] + action_generator: Optional[PotentialActionGeneratorAction] + + def get_action_type(self) -> ActionType: + matches = [v for v in ActionType if v in self and self[v]] + if len(matches) != 1: + ActionType.raise_invalid_action_declaration() + return ActionType(matches[0]) diff --git a/monitoring/uss_qualifier/action_generators/documentation/documentation.py b/monitoring/uss_qualifier/action_generators/documentation/documentation.py new file mode 100644 index 0000000000..ec1193f4c5 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/documentation/documentation.py @@ -0,0 +1,78 @@ +from typing import List, Union + +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, + action_generator_specification_type, +) +from monitoring.uss_qualifier.action_generators.definitions import ( + ActionGeneratorDefinition, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, + PotentialTestSuiteAction, + PotentialActionGeneratorAction, +) +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteActionDeclaration, + ActionType, +) + + +def list_potential_actions_for_action_generator_definition( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction] +) -> List[PotentialGeneratedAction]: + action_generator_type = action_generator_type_from_name( + generator_def.generator_type + ) + specification_type = action_generator_specification_type(action_generator_type) + if specification_type is not None: + spec = ImplicitDict.parse(generator_def.specification, specification_type) + else: + spec = None + return action_generator_type.list_potential_actions(spec) + + +def list_potential_actions_for_action_declaration( + declaration: TestSuiteActionDeclaration, +) -> List[PotentialGeneratedAction]: + action_type = declaration.get_action_type() + if action_type == ActionType.TestScenario: + return [ + PotentialGeneratedAction( + test_scenario=PotentialTestScenarioAction( + scenario_type=declaration.test_scenario.scenario_type + ) + ) + ] + elif action_type == ActionType.TestSuite: + if "suite_type" in declaration.test_suite and declaration.test_suite.suite_type: + return [ + PotentialGeneratedAction( + test_suite=PotentialTestSuiteAction( + suite_type=declaration.test_suite.suite_type + ) + ) + ] + elif ( + "suite_definition" in declaration.test_suite + and declaration.test_suite.suite_definition + ): + return [ + PotentialGeneratedAction( + test_suite=PotentialTestSuiteAction( + suite_definition=declaration.test_suite.suite_definition + ) + ) + ] + elif action_type == ActionType.ActionGenerator: + return [ + PotentialGeneratedAction( + action_generator=PotentialActionGeneratorAction( + generator_type=declaration.action_generator.generator_type + ) + ) + ] + else: + raise NotImplementedError(f"Action type {action_type} is not supported") diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py index f81899c677..38d2ddf54e 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py +++ b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py @@ -3,6 +3,17 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, + action_generator_specification_type, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource @@ -14,7 +25,10 @@ ResourceType, ) -from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteActionDeclaration, + ActionType, +) from monitoring.uss_qualifier.suites.suite import ( ActionGenerator, TestSuiteAction, @@ -43,6 +57,14 @@ class FlightPlannerCombinations( _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: FlightPlannerCombinationsSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: FlightPlannerCombinationsSpecification, diff --git a/monitoring/uss_qualifier/action_generators/repetition/repeat.py b/monitoring/uss_qualifier/action_generators/repetition/repeat.py index c282a73693..a2f3256af2 100644 --- a/monitoring/uss_qualifier/action_generators/repetition/repeat.py +++ b/monitoring/uss_qualifier/action_generators/repetition/repeat.py @@ -1,6 +1,12 @@ from typing import Dict, List, Optional from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.resource import ResourceType @@ -26,6 +32,14 @@ class Repeat(ActionGenerator[RepeatSpecification]): _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: RepeatSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: RepeatSpecification, diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index d6bb83e63c..cb4371d828 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -7,6 +7,7 @@ from implicitdict import ImplicitDict, StringBasedDateTime from monitoring.monitorlib import fetch, inspection +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ( TestConfiguration, @@ -18,6 +19,7 @@ JSONPathExpression, ) from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName class FailedCheck(ImplicitDict): @@ -211,7 +213,7 @@ class TestScenarioReport(ImplicitDict): name: str """Name of this test scenario""" - scenario_type: str + scenario_type: TestScenarioTypeName """Type of this test scenario""" documentation_url: str @@ -295,7 +297,7 @@ def participant_ids(self) -> Set[ParticipantID]: class ActionGeneratorReport(ImplicitDict): - generator_type: str + generator_type: GeneratorTypeName """Type of action generator""" actions: List[TestSuiteActionReport] diff --git a/monitoring/uss_qualifier/scenarios/README.md b/monitoring/uss_qualifier/scenarios/README.md index 75a5986a26..09b920b3eb 100644 --- a/monitoring/uss_qualifier/scenarios/README.md +++ b/monitoring/uss_qualifier/scenarios/README.md @@ -87,6 +87,8 @@ A check should document the requirement(s) violated if the check fails. Require Any requirements identified (e.g., `**astm.f3411.v19.NET0420**`) must be documented as well. See [the requirements documentation](../requirements/README.md) for more information. +If the text of this section includes `TODO:`, then the check will be indicated as in development rather than complete. Documentation of intended checks with, e.g., `TODO: Implement` prior to the start of Python development is highly encouraged. + ### Cleanup phase If a test scenario wants to perform a cleanup procedure follow any non-error termination of the rest of the scenario, it must: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 598b3b52c0..3aded3234e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -52,3 +52,19 @@ of the durations for the subsequent display data queries do not exceed the respe **[astm.f3411.v19.NET0260-a](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles of the durations for the replies to requested flights in an area do not exceed the respective thresholds `NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). + +## Mock USS interactions evaluation test case + +In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify +compliance with requirements. + +### Get mock USS interactions test step + +### Evaluate mock USS interactions test step + +#### No large Display Provider queries check + +If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed +area requested, then that participant will have violated **[astm.f3411.v19.NET0240](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement this check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 94e1b5482c..8818d8e14e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -52,3 +52,19 @@ of the durations for the subsequent display data queries do not exceed the respe **[astm.f3411.v22a.NET0260-a](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles of the durations for the replies to requested flights in an area do not exceed the respective thresholds `NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). + +## Mock USS interactions evaluation test case + +In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify +compliance with requirements. + +### Get mock USS interactions test step + +### Evaluate mock USS interactions test step + +#### No large Display Provider queries check + +If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed +area requested, then that participant will have violated **[astm.f3411.v22a.NET0240](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement this check diff --git a/monitoring/uss_qualifier/scenarios/definitions.py b/monitoring/uss_qualifier/scenarios/definitions.py index 5672c40b10..90d8c556fc 100644 --- a/monitoring/uss_qualifier/scenarios/definitions.py +++ b/monitoring/uss_qualifier/scenarios/definitions.py @@ -5,9 +5,13 @@ from monitoring.uss_qualifier.resources.definitions import ResourceID +TestScenarioTypeName = str +"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the `uss_qualifier` module""" + + class TestScenarioDeclaration(ImplicitDict): - scenario_type: FileReference - """Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder""" + scenario_type: TestScenarioTypeName + """Type of test scenario.""" resources: Optional[Dict[ResourceID, ResourceID]] """Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite. diff --git a/monitoring/uss_qualifier/scenarios/documentation/definitions.py b/monitoring/uss_qualifier/scenarios/documentation/definitions.py index 1cde8ac14d..ef69179a87 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/definitions.py +++ b/monitoring/uss_qualifier/scenarios/documentation/definitions.py @@ -8,6 +8,7 @@ class TestCheckDocumentation(ImplicitDict): name: str url: Optional[str] = None applicable_requirements: List[RequirementID] + has_todo: bool class TestStepDocumentation(ImplicitDict): diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index cdec8572ba..a90227a53c 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -10,6 +10,8 @@ from monitoring import uss_qualifier as uss_qualifier_module from monitoring.monitorlib import versioning from monitoring.monitorlib.inspection import fullname, get_module_object_by_name +from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestStepDocumentation, TestCheckDocumentation, @@ -38,6 +40,8 @@ def _text_of(value) -> str: result += _text_of(child) return result elif isinstance(value, marko.inline.InlineElement): + if isinstance(value, marko.inline.LineBreak): + return "\n" if isinstance(value.children, str): return value.children result = "" @@ -73,17 +77,22 @@ def _parse_test_check( ) -> TestCheckDocumentation: name = _text_of(values[0])[0 : -len(TEST_CHECK_SUFFIX)] url = _url_of(doc_filename + anchors[values[0]]) + has_todo = False reqs: List[str] = [] c = 1 while c < len(values): if isinstance(values[c], marko.block.Paragraph): + if "TODO:" in _text_of(values[c]): + has_todo = True for child in values[c].children: if isinstance(child, marko.inline.StrongEmphasis): - reqs.append(_text_of(child)) + reqs.append(RequirementID(_text_of(child))) c += 1 - return TestCheckDocumentation(name=name, url=url, applicable_requirements=reqs) + return TestCheckDocumentation( + name=name, url=url, applicable_requirements=reqs, has_todo=has_todo + ) def _get_linked_test_step( @@ -310,6 +319,8 @@ def get_documentation(scenario: Type) -> TestScenarioDocumentation: return getattr(scenario, DOC_CACHE_ATTRIBUTE) -def get_documentation_by_name(scenario_type_name: str) -> TestScenarioDocumentation: +def get_documentation_by_name( + scenario_type_name: TestScenarioTypeName, +) -> TestScenarioDocumentation: scenario_type = get_module_object_by_name(uss_qualifier_module, scenario_type_name) return get_documentation(scenario_type) diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py index 6024aa62b1..81f5fb4928 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py @@ -92,6 +92,7 @@ def _plan_flights(self): name=c.name, url=c.url, applicable_requirements=row.requirement_ids, + has_todo=False, ) for c in checks ] diff --git a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py index efb0adc6ee..1ba899cd98 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py +++ b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py @@ -63,6 +63,7 @@ def _map_query(self): name=check_name, url=check_url, applicable_requirements=row.requirement_ids, + has_todo=False, ) doc = TestStepDocumentation( name=row.geospatial_check_id, diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index b3901e938e..78a0791fb3 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -24,7 +24,10 @@ ParticipantID, PassedCheck, ) -from monitoring.uss_qualifier.scenarios.definitions import TestScenarioDeclaration +from monitoring.uss_qualifier.scenarios.definitions import ( + TestScenarioDeclaration, + TestScenarioTypeName, +) from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestScenarioDocumentation, TestCaseDocumentation, @@ -159,7 +162,7 @@ def record_passed( self._step_report.passed_checks.append(passed_check) -def get_scenario_type_by_name(scenario_type_name: str) -> Type: +def get_scenario_type_by_name(scenario_type_name: TestScenarioTypeName) -> Type: inspection.import_submodules(scenarios_module) scenario_type = inspection.get_module_object_by_name( parent_module=uss_qualifier_module, object_name=scenario_type_name @@ -361,7 +364,7 @@ def check( check_list = ", ".join(available_checks) if self._allow_undocumented_checks: check_documentation = TestCheckDocumentation( - name=name, applicable_requirements=[] + name=name, applicable_requirements=[], has_todo=False ) else: raise RuntimeError( diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index b91c6ea824..274f6f92e9 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -4,6 +4,273 @@ ## Actions -1. Action generator: `action_generators.astm.f3411.ForEachDSS` +1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) + 1. Suite: [DSS instance validator](f3411_19_suite1.md) ([in-suite definition](f3411_19.yaml)) 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) +3. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v19A2-6-1,1aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3cImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,3dImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-19 NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-19 NetRID DSS interoperability
DSS0030ImplementedASTM NetRID nominal behavior
DSS0070ImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130ImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-19 NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-19 NetRID DSS interoperability
DSS0210ImplementedASTM F3411-19 NetRID DSS interoperability
NET0240PlannedASTM F3411-19 NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-19 NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-19 NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-19 NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID nominal behavior
astm.f3411.v22aDSS0030ImplementedASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
interuss.automated_testing.rid.injectionDeleteTestSuccessImplementedASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultImplementedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md new file mode 100644 index 0000000000..2fe606005f --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md @@ -0,0 +1,34 @@ + +# DSS instance validator test suite +Defined in [parent suite](f3411_19.md) [`suites.astm.netrid.f3411_19`](./f3411_19.yaml) + +## Actions + +1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aDSS0030ImplementedASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 8dd20db5b5..e466936d21 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -4,6 +4,307 @@ ## Actions -1. Action generator: `action_generators.astm.f3411.ForEachDSS` +1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) + 1. Suite: [DSS instance validator](f3411_22a_suite1.md) ([in-suite definition](f3411_22a.yaml)) 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) +3. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aA2-6-1,1aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0210ImplementedASTM F3411-22a NetRID DSS interoperability
NET0240PlannedASTM F3411-22a NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260,Table1,1ImplementedASTM NetRID nominal behavior
NET0260,Table1,1aImplementedASTM NetRID nominal behavior
NET0260,Table1,23ImplementedASTM NetRID nominal behavior
NET0260,Table1,24ImplementedASTM NetRID nominal behavior
NET0260,Table1,25ImplementedASTM NetRID nominal behavior
NET0260,Table1,26ImplementedASTM NetRID nominal behavior
NET0260,Table1,7ImplementedASTM NetRID nominal behavior
NET0260,Table1,9ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-22a NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-22a NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-22a NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
interuss.automated_testing.rid.injectionDeleteTestSuccessImplementedASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultImplementedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md new file mode 100644 index 0000000000..0c90a2bfb6 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md @@ -0,0 +1,40 @@ + +# DSS instance validator test suite +Defined in [parent suite](f3411_22a.md) [`suites.astm.netrid.f3411_22a`](./f3411_22a.yaml) + +## Actions + +1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) +2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aDSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
NET0710ImplementedASTM NetRID DSS: Simple ISA
diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 62a6f5c093..d27cf50d52 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -4,6 +4,127 @@ ## Actions -1. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` -2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` -3. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +1. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) +2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) +3. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3548.v21DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss.automated_testing.flight_planningClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index ca453656d1..c708113606 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -1,18 +1,59 @@ +from __future__ import annotations import glob import inspect import os -from typing import Iterator, Optional +from dataclasses import dataclass +from typing import Iterator, Optional, List, Union, Dict from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, +) +from monitoring.uss_qualifier.action_generators.definitions import ( + ActionGeneratorDefinition, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialActionGeneratorAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_generator_definition, +) from monitoring.uss_qualifier.fileio import ( load_dict_with_references, get_package_name, resolve_filename, + FileReference, +) +from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.requirements.documentation import get_requirement +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName +from monitoring.uss_qualifier.scenarios.documentation.definitions import ( + TestScenarioDocumentation, + TestCheckDocumentation, +) +from monitoring.uss_qualifier.scenarios.documentation.parsing import ( + get_documentation, + get_documentation_by_name, ) -from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name -from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition, ActionType +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteDefinition, + ActionType, + TestSuiteActionDeclaration, +) +from monitoring.uss_qualifier.suites.suite import TestSuiteAction + + +@dataclass +class TestSuiteRenderContext(object): + parent_yaml_file: str + parent_doc_file: str + base_path: str + list_index: int + indent: int + test_suites: Dict[str, str] def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: @@ -28,63 +69,360 @@ def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: yield suite -def make_test_suite_documentation(test_suite_yaml_file: str) -> str: +def make_test_suite_documentation( + suite_def: TestSuiteDefinition, + suite_yaml_file: str, + suite_doc_file: str, + parent_suite_doc: Optional[str] = None, +) -> Dict[str, str]: + test_suites: Dict[str, str] = {} + lines = [] - suite_def: TestSuiteDefinition = ImplicitDict.parse( - load_dict_with_references("file://" + test_suite_yaml_file), TestSuiteDefinition - ) lines.append( "" ) lines.append(f"# {suite_def.name} test suite") - local_path = os.path.split(test_suite_yaml_file)[-1] - lines.append(f"[`{get_package_name(test_suite_yaml_file)}`](./{local_path})") + local_path = os.path.split(suite_yaml_file)[-1] + if parent_suite_doc is None: + prefix = "" + else: + parent_rel_path = os.path.relpath( + parent_suite_doc, start=os.path.dirname(suite_doc_file) + ) + prefix = f"Defined in [parent suite]({parent_rel_path}) " + lines.append(f"{prefix}[`{get_package_name(suite_yaml_file)}`](./{local_path})") lines.append("") lines.append("## Actions") lines.append("") - base_path = os.path.dirname(test_suite_yaml_file) + base_path = os.path.dirname(suite_yaml_file) + i = 0 for i, action in enumerate(suite_def.actions): - action_type = action.get_action_type() - if action_type == ActionType.TestScenario: - scenario_type = get_scenario_type_by_name( - action.test_scenario.scenario_type + lines.extend( + _render_action( + action, + TestSuiteRenderContext( + parent_yaml_file=suite_yaml_file, + parent_doc_file=suite_doc_file, + base_path=base_path, + list_index=i + 1, + indent=0, + test_suites=test_suites, + ), ) - py_rel_path = os.path.relpath(inspect.getfile(scenario_type), base_path) - scenario_doc = get_documentation(scenario_type) - doc_rel_path = os.path.relpath(scenario_doc.local_path, start=base_path) - lines.append( - f"{i + 1}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{action.test_scenario.scenario_type}`]({py_rel_path}))" + ) + if ( + "report_evaluation_scenario" in suite_def + and suite_def.report_evaluation_scenario + ): + lines.extend( + _render_scenario( + suite_def.report_evaluation_scenario.scenario_type, + TestSuiteRenderContext( + parent_yaml_file=suite_yaml_file, + parent_doc_file=suite_doc_file, + base_path=base_path, + list_index=i + 1, + indent=0, + test_suites=test_suites, + ), ) - elif action_type == ActionType.TestSuite: - if "suite_type" in action.test_suite and action.test_suite.suite_type: - suite_def = ImplicitDict.parse( - load_dict_with_references(action.test_suite.suite_type), - TestSuiteDefinition, - ) - suite_path = resolve_filename(action.test_suite.suite_type) - suite_rel_path = os.path.relpath(suite_path, start=base_path) - doc_path = os.path.splitext(suite_path)[0] + ".md" - doc_rel_path = os.path.relpath(doc_path, start=base_path) - lines.append( - f"{i + 1}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{action.test_suite.suite_type}`]({suite_rel_path}))" + ) + lines.append("") + + lines.append("## Checked requirements") + lines.append("") + reqs = _collect_requirements_from_suite_def(suite_def) + if not reqs: + lines.append( + "_This test suite documentation does not indicate that any requirements are checked._" + ) + else: + # Use an HTML table rather than Markdown table to enabled advanced features like spans + lines.append("") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + + req_ids_by_package: Dict[str, List[RequirementID]] = {} + for req_id, req in reqs.items(): + package = req_ids_by_package.get(req_id.package(), []) + if req_id not in package: + package.append(req_id) + req_ids_by_package[req_id.package()] = package + + for package in sorted(req_ids_by_package): + req_md_path = os.path.relpath( + req_ids_by_package[package][0].md_file_path(), start=base_path + ) + package_line = f' ' + for req_id in sorted(req_ids_by_package[package]): + req_text = f'{req_id.requirement_name()}' + + has_todo = False + has_complete = False + scenarios = {} + for checked_in in reqs[req_id].checked_in: + if checked_in.scenario.local_path not in scenarios: + scenarios[checked_in.scenario.name] = checked_in.scenario + if checked_in.check.has_todo: + has_todo = True + else: + has_complete = True + if has_complete and not has_todo: + status_text = "Implemented" + elif has_todo and not has_complete: + status_text = "Planned" + elif has_todo and has_complete: + status_text = "In progress" + else: + status_text = "Not implemented" + checked_in = list( + f'{scenarios[s].name}' + for s in sorted(scenarios) ) - elif "suite_definition" in action.test_suite and action.suite_definition: - # TODO: Generate additional test suite documentation for in-suite suite definition - lines.append(f"{i + 1}. Suite: ") + checked_in_text = f"{'
'.join(checked_in)}" + + lines.append(" ") + if package_line: + lines.append(package_line) + package_line = None + lines.append(f" ") + lines.append(f" ") + lines.append(f" ") + lines.append(" ") + lines.append("
PackageRequirementStatusChecked in
{package}
{req_text}{status_text}{checked_in_text}
") + lines.append("") + + test_suites[suite_doc_file] = "\n".join(lines) + return test_suites + + +def _render_scenario( + scenario_type_name: TestScenarioTypeName, context: TestSuiteRenderContext +) -> List[str]: + lines = [] + scenario_type = get_scenario_type_by_name(scenario_type_name) + py_rel_path = os.path.relpath(inspect.getfile(scenario_type), context.base_path) + scenario_doc = get_documentation(scenario_type) + doc_rel_path = os.path.relpath(scenario_doc.local_path, start=context.base_path) + lines.append( + f"{' ' * context.indent}{context.list_index}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{scenario_type_name}`]({py_rel_path}))" + ) + return lines + + +def _render_suite_by_type( + suite_type: FileReference, context: TestSuiteRenderContext +) -> List[str]: + lines = [] + suite_def = ImplicitDict.parse( + load_dict_with_references(suite_type), + TestSuiteDefinition, + ) + suite_path = resolve_filename(suite_type) + suite_rel_path = os.path.relpath(suite_path, start=context.base_path) + doc_path = os.path.splitext(suite_path)[0] + ".md" + doc_rel_path = os.path.relpath(doc_path, start=context.base_path) + lines.append( + f"{' ' * context.indent}{context.list_index}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{suite_type}`]({suite_rel_path}))" + ) + return lines + + +def _render_suite_by_definition( + suite_def: TestSuiteDefinition, context: TestSuiteRenderContext +) -> List[str]: + doc_path = ( + os.path.splitext(context.parent_doc_file)[0] + f"_suite{context.list_index}.md" + ) + new_docs = make_test_suite_documentation( + suite_def, context.parent_yaml_file, doc_path, context.parent_doc_file + ) + + for k, v in new_docs.items(): + context.test_suites[k] = v + + doc_rel_path = os.path.relpath(doc_path, context.base_path) + parent_rel_path = os.path.relpath(context.parent_yaml_file, start=context.base_path) + return [ + f"{' ' * context.indent}{context.list_index}. Suite: [{suite_def.name}]({doc_rel_path}) ([in-suite definition]({parent_rel_path}))" + ] + + +def _render_action_generator( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction], + context: TestSuiteRenderContext, +) -> List[str]: + lines = [] + action_generator_type = action_generator_type_from_name( + generator_def.generator_type + ) + py_rel_path = os.path.relpath( + inspect.getfile(action_generator_type), start=context.base_path + ) + lines.append( + f"{' ' * context.indent}{context.list_index}. Action generator: [`{generator_def.generator_type}`]({py_rel_path})" + ) + potential_actions = list_potential_actions_for_action_generator_definition( + generator_def + ) + for j, potential_action in enumerate(potential_actions): + lines.extend( + _render_action( + potential_action, + TestSuiteRenderContext( + parent_yaml_file=context.parent_yaml_file, + parent_doc_file=context.parent_doc_file, + base_path=context.base_path, + list_index=j + 1, + indent=context.indent + 4, + test_suites=context.test_suites, + ), + ) + ) + return lines + + +def _render_action( + action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction], + context: TestSuiteRenderContext, +) -> List[str]: + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + return _render_scenario(action.test_scenario.scenario_type, context) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + return _render_suite_by_type(action.test_suite.suite_type, context) + elif ( + "suite_definition" in action.test_suite + and action.test_suite.suite_definition + ): + return _render_suite_by_definition( + action.test_suite.suite_definition, context + ) + else: + raise ValueError( + f"Test suite action {context.list_index} missing suite type or definition" + ) + elif action_type == ActionType.ActionGenerator: + return _render_action_generator(action.action_generator, context) + else: + raise NotImplementedError(f"Unsupported test suite action type: {action_type}") + + +@dataclass +class SuiteLocation(object): + scenario: TestScenarioDocumentation + check: TestCheckDocumentation + + +@dataclass +class RequirementInSuite(object): + checked_in: List[SuiteLocation] + + def extend(self, other: RequirementInSuite): + self.checked_in.extend(other.checked_in) + + +def _collect_requirements_from_suite_def( + suite_def: TestSuiteDefinition, +) -> Dict[RequirementID, RequirementInSuite]: + reqs: Dict[RequirementID, RequirementInSuite] = {} + + def combine(new_reqs: Dict[RequirementID, RequirementInSuite]) -> None: + for req_id, req in new_reqs.items(): + if req_id not in reqs: + reqs[req_id] = req else: - raise ValueError( - f"Test suite action {i + 1} missing suite type or definition in {test_suite_yaml_file}" - ) - elif action_type == ActionType.ActionGenerator: - # TODO: Add documentation for action generators - lines.append( - f"{i + 1}. Action generator: `{action.action_generator.generator_type}`" + reqs[req_id].extend(req) + + for action in suite_def.actions: + combine(_collect_requirements_from_action(action)) + if ( + "report_evaluation_scenario" in suite_def + and suite_def.report_evaluation_scenario + ): + combine( + _collect_requirements_from_scenario( + suite_def.report_evaluation_scenario.scenario_type + ) + ) + return reqs + + +def _collect_requirements_from_action( + action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction] +) -> Dict[RequirementID, RequirementInSuite]: + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + return _collect_requirements_from_scenario(action.test_scenario.scenario_type) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + suite_def = ImplicitDict.parse( + load_dict_with_references(action.test_suite.suite_type), + TestSuiteDefinition, + ) + return _collect_requirements_from_suite_def(suite_def) + elif ( + "suite_definition" in action.test_suite + and action.test_suite.suite_definition + ): + return _collect_requirements_from_suite_def( + action.test_suite.suite_definition ) else: - raise NotImplementedError( - f"Unsupported test suite action type: {action_type}" + raise ValueError( + "Neither suite_type nor suite_definition specified in test_suite action" ) + elif action_type == ActionType.ActionGenerator: + return _collect_requirements_from_action_generator(action.action_generator) + else: + raise NotImplementedError( + f"Test suite action type {action_type} not yet supported" + ) - lines.append("") - return "\n".join(lines) + +def _collect_requirements_from_scenario( + scenario_type: TestScenarioTypeName, +) -> Dict[RequirementID, RequirementInSuite]: + docs = get_documentation_by_name(scenario_type) + reqs: Dict[RequirementID, RequirementInSuite] = {} + + def add_req(req_id: RequirementID, check: TestCheckDocumentation) -> None: + req = reqs.get(req_id, RequirementInSuite(checked_in=[])) + req.checked_in.append(SuiteLocation(scenario=docs, check=check)) + reqs[req_id] = req + + for case in docs.cases: + for step in case.steps: + for check in step.checks: + for req_id in check.applicable_requirements: + add_req(req_id, check) + if "cleanup" in docs and docs.cleanup: + for check in docs.cleanup.checks: + for req_id in check.applicable_requirements: + add_req(req_id, check) + return reqs + + +def _collect_requirements_from_action_generator( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction] +) -> Dict[RequirementID, RequirementInSuite]: + potential_actions = list_potential_actions_for_action_generator_definition( + generator_def + ) + + reqs: Dict[RequirementID, RequirementInSuite] = {} + for potential_action in potential_actions: + new_reqs = _collect_requirements_from_action(potential_action) + for req_id, req in new_reqs.items(): + if req_id not in reqs: + reqs[req_id] = req + else: + reqs[req_id].extend(req) + + return reqs diff --git a/monitoring/uss_qualifier/suites/documentation/format_documentation.py b/monitoring/uss_qualifier/suites/documentation/format_documentation.py index 10b3ea2a5a..0859588a5d 100644 --- a/monitoring/uss_qualifier/suites/documentation/format_documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/format_documentation.py @@ -2,6 +2,11 @@ import os import sys +from implicitdict import ImplicitDict +from monitoring.monitorlib.inspection import import_submodules +from monitoring.uss_qualifier import scenarios, suites, action_generators +from monitoring.uss_qualifier.fileio import load_dict_with_references +from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition from monitoring.uss_qualifier.suites.documentation.documentation import ( find_test_suites, make_test_suite_documentation, @@ -9,10 +14,24 @@ def main(lint: bool) -> int: - changes = False + import_submodules(scenarios) + import_submodules(suites) + import_submodules(action_generators) + + test_suite_docs = {} for suite_yaml_file in find_test_suites(): - suite_doc_content = make_test_suite_documentation(suite_yaml_file) + suite_def: TestSuiteDefinition = ImplicitDict.parse( + load_dict_with_references("file://" + suite_yaml_file), TestSuiteDefinition + ) suite_doc_file = os.path.splitext(suite_yaml_file)[0] + ".md" + new_docs = make_test_suite_documentation( + suite_def, suite_yaml_file, suite_doc_file + ) + for k, v in new_docs.items(): + test_suite_docs[k] = v + + changes = False + for suite_doc_file, suite_doc_content in test_suite_docs.items(): if os.path.exists(suite_doc_file): with open(suite_doc_file, "r") as f: existing_content = f.read() diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index a5fe6c401d..3968595b3f 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -7,3 +7,121 @@ 1. Scenario: [Start message signing](../../../scenarios/faa/uft/message_signing_start.md) ([`scenarios.faa.uft.StartMessageSigningReport`](../../../scenarios/faa/uft/message_signing_start.py)) 2. Suite: [ASTM F3548-21](../../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../../astm/utm/f3548_21.yaml)) 3. Scenario: [Finalize message signing](../../../scenarios/faa/uft/message_signing_finalize.md) ([`scenarios.faa.uft.FinalizeMessageSigningReport`](../../../scenarios/faa/uft/message_signing_finalize.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3548.v21DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss.automated_testing.flight_planningClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 25bdb21bce..a2800dee46 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -5,4 +5,123 @@ ## Actions 1. Suite: [ASTM F3548-21](../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../astm/utm/f3548_21.yaml)) -2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Flight authorisation validation](../../scenarios/uspace/flight_auth/validation.md) ([`scenarios.uspace.flight_auth.Validation`](../../scenarios/uspace/flight_auth/validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3548.v21DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss.automated_testing.flight_planningClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
diff --git a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md index 910b3700fb..22d7140529 100644 --- a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md +++ b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md @@ -5,3 +5,7 @@ ## Actions 1. Scenario: [EUROCAE ED-269 UAS geographical zone model](../../scenarios/eurocae/ed269/source_data_model.md) ([`scenarios.eurocae.ed269.source_data_model.SourceDataModelValidation`](../../scenarios/eurocae/ed269/source_data_model.py)) + +## Checked requirements + +_This test suite documentation does not indicate that any requirements are checked._ diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 1cfec80a32..7d84eea167 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -5,3 +5,302 @@ ## Actions 1. Suite: [ASTM F3411-22a](../astm/netrid/f3411_22a.md) ([`suites.astm.netrid.f3411_22a`](../astm/netrid/f3411_22a.yaml)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aA2-6-1,1aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0210ImplementedASTM F3411-22a NetRID DSS interoperability
NET0240PlannedASTM F3411-22a NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260,Table1,1ImplementedASTM NetRID nominal behavior
NET0260,Table1,1aImplementedASTM NetRID nominal behavior
NET0260,Table1,23ImplementedASTM NetRID nominal behavior
NET0260,Table1,24ImplementedASTM NetRID nominal behavior
NET0260,Table1,25ImplementedASTM NetRID nominal behavior
NET0260,Table1,26ImplementedASTM NetRID nominal behavior
NET0260,Table1,7ImplementedASTM NetRID nominal behavior
NET0260,Table1,9ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-22a NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-22a NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-22a NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
interuss.automated_testing.rid.injectionDeleteTestSuccessImplementedASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultImplementedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 6598ea51be..ba7e196d4f 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -6,3 +6,409 @@ 1. Suite: [U-space flight authorisation](flight_auth.md) ([`suites.uspace.flight_auth`](flight_auth.yaml)) 2. Suite: [U-Space network identification](network_identification.md) ([`suites.uspace.network_identification`](network_identification.yaml)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementStatusChecked in
astm.f3411.v22aA2-6-1,1aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ImplementedASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0030ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ImplementedASTM NetRID DSS: Subscription Validation
DSS0060ImplementedASTM NetRID DSS: Subscription Validation
DSS0070ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130ImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cImplementedASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dImplementedASTM F3411-22a NetRID DSS interoperability
DSS0210ImplementedASTM F3411-22a NetRID DSS interoperability
NET0240PlannedASTM F3411-22a NetRID aggregate checks
NET0260ImplementedASTM NetRID nominal behavior
NET0260,Table1,1ImplementedASTM NetRID nominal behavior
NET0260,Table1,1aImplementedASTM NetRID nominal behavior
NET0260,Table1,23ImplementedASTM NetRID nominal behavior
NET0260,Table1,24ImplementedASTM NetRID nominal behavior
NET0260,Table1,25ImplementedASTM NetRID nominal behavior
NET0260,Table1,26ImplementedASTM NetRID nominal behavior
NET0260,Table1,7ImplementedASTM NetRID nominal behavior
NET0260,Table1,9ImplementedASTM NetRID nominal behavior
NET0260-aImplementedASTM F3411-22a NetRID aggregate checks
NET0270ImplementedASTM NetRID nominal behavior
NET0290ImplementedASTM NetRID nominal behavior
NET0420ImplementedASTM F3411-22a NetRID aggregate checks
NET0430ImplementedASTM NetRID nominal behavior
NET0440ImplementedASTM F3411-22a NetRID aggregate checks
NET0470ImplementedASTM NetRID nominal behavior
NET0480ImplementedASTM NetRID nominal behavior
NET0490ImplementedASTM NetRID nominal behavior
NET0500ImplementedASTM NetRID nominal behavior
NET0610ImplementedASTM NetRID nominal behavior
NET0710ImplementedASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
astm.f3548.v21DSS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0310ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
GEN0500ImplementedValidation of operational intents
OPIN0015ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0020ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0025ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
OPIN0030ImplementedValidation of operational intents
OPIN0040ImplementedValidation of operational intents
SCD0015ImplementedNominal planning: conflict with higher priority
SCD0020ImplementedNominal planning: conflict with higher priority
SCD0025ImplementedNominal planning: conflict with higher priority
SCD0030ImplementedNominal planning: conflict with higher priority
SCD0035ImplementedNominal planning: not permitted conflict with equal priority
SCD0040ImplementedNominal planning: not permitted conflict with equal priority
SCD0045ImplementedNominal planning: not permitted conflict with equal priority
SCD0050ImplementedNominal planning: not permitted conflict with equal priority
USS0005ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
USS0105ImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss.automated_testing.flight_planningClearAreaImplementedNominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
DeleteFlightSuccessImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
ExpectedBehaviorImplementedFlight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents
interuss.automated_testing.rid.injectionDeleteTestSuccessImplementedASTM NetRID nominal behavior
ExpectedBehaviorImplementedASTM NetRID nominal behavior
UpsertTestResultImplementedASTM NetRID nominal behavior
UpsertTestSuccessImplementedASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessImplementedASTM NetRID nominal behavior
UniqueFlightsImplementedASTM NetRID nominal behavior
diff --git a/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json b/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json index 60d2424977..26e32cbfaf 100644 --- a/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json +++ b/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json @@ -1,35 +1,35 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "resources": { - "type": [ - "object", - "null" - ], + "additionalProperties": { + "type": "string" + }, + "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource .", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" } }, - "additionalProperties": { - "type": "string" - }, - "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource ." + "type": [ + "object", + "null" + ] }, "scenario_type": { - "type": "string", - "description": "Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder" + "description": "Type of test scenario.", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", - "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "required": [ "scenario_type" - ] + ], + "type": "object" } \ No newline at end of file