From f1cacce1f12796f97ba3c38452426e23b8b22b9d Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Sat, 9 Sep 2023 06:37:05 +0000 Subject: [PATCH 1/8] Generate test suite documentation for action generators --- .../action_generators/action_generator.py | 80 +++++++--- .../astm/f3411/for_each_dss.py | 14 ++ .../action_generators/definitions.py | 2 +- .../documentation/__init__.py | 0 .../documentation/definitions.py | 40 +++++ .../documentation/documentation.py | 78 +++++++++ .../flight_planning/planner_combinations.py | 24 ++- .../action_generators/repetition/repeat.py | 14 ++ monitoring/uss_qualifier/reports/report.py | 6 +- .../uss_qualifier/scenarios/definitions.py | 8 +- .../uss_qualifier/scenarios/scenario.py | 7 +- .../suites/astm/netrid/f3411_19.md | 3 +- .../suites/astm/netrid/f3411_22a.md | 3 +- .../uss_qualifier/suites/astm/utm/f3548_21.md | 9 +- .../suites/documentation/documentation.py | 148 +++++++++++++----- .../suites/uspace/flight_auth.md | 3 +- .../definitions/TestScenarioDeclaration.json | 36 ++--- 17 files changed, 381 insertions(+), 94 deletions(-) create mode 100644 monitoring/uss_qualifier/action_generators/documentation/__init__.py create mode 100644 monitoring/uss_qualifier/action_generators/documentation/definitions.py create mode 100644 monitoring/uss_qualifier/action_generators/documentation/documentation.py diff --git a/monitoring/uss_qualifier/action_generators/action_generator.py b/monitoring/uss_qualifier/action_generators/action_generator.py index 71694d5547..5505dd8d04 100644 --- a/monitoring/uss_qualifier/action_generators/action_generator.py +++ b/monitoring/uss_qualifier/action_generators/action_generator.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod import inspect -from typing import Generic, Dict, Optional, TypeVar +from typing import Generic, Dict, Optional, TypeVar, List, Type from implicitdict import ImplicitDict from monitoring import uss_qualifier as uss_qualifier_module @@ -12,6 +12,10 @@ from monitoring.uss_qualifier.action_generators.definitions import ( ActionGeneratorSpecificationType, ActionGeneratorDefinition, + GeneratorTypeName, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, ) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID @@ -45,32 +49,35 @@ def run_next_action(self) -> Optional[TestSuiteActionReport]: "A concrete action generator must implement `actions` method" ) + @classmethod + def list_potential_actions( + cls, specification: Optional[ActionGeneratorSpecificationType] + ) -> List[PotentialGeneratedAction]: + """Enumerate the potential actions that may be run by an instance of this generator type. + + Concrete subclasses of ActionGenerator must implement a classmethod that shadows this one according to this + specification. + + Args: + specification: A serializable (subclass of implicitdict.ImplicitDict) specification for how to create the + action generator instance, or None if the action generator type does not need a specification. + + Returns: All potential actions that may be generated by this generator, depending on the resources provided. + """ + raise NotImplementedError( + "A concrete action generator must implement `list_potential_actions` classmethod" + ) + @staticmethod def make_from_definition( definition: ActionGeneratorDefinition, resources: Dict[ResourceID, ResourceType] ) -> ActionGeneratorType: - from monitoring.uss_qualifier import ( - action_generators as action_generators_module, + action_generator_type = action_generator_type_from_name( + definition.generator_type ) + specification_type = action_generator_specification_type(action_generator_type) - import_submodules(action_generators_module) - action_generator_type = get_module_object_by_name( - parent_module=uss_qualifier_module, - object_name=definition.generator_type, - ) - if not issubclass(action_generator_type, ActionGenerator): - raise NotImplementedError( - "Action generator type {} is not a subclass of the ActionGenerator base class".format( - action_generator_type.__name__ - ) - ) - constructor_signature = inspect.signature(action_generator_type.__init__) - specification_type = None constructor_args = {} - for arg_name, arg in constructor_signature.parameters.items(): - if arg_name == "specification": - specification_type = arg.annotation - break if specification_type is not None: constructor_args["specification"] = ImplicitDict.parse( definition.specification, specification_type @@ -82,3 +89,36 @@ def make_from_definition( ActionGeneratorType = TypeVar("ActionGeneratorType", bound=ActionGenerator) + + +def action_generator_type_from_name( + action_generator_type_name: GeneratorTypeName, +) -> Type[ActionGenerator]: + from monitoring.uss_qualifier import ( + action_generators as action_generators_module, + ) + + import_submodules(action_generators_module) + action_generator_type = get_module_object_by_name( + parent_module=uss_qualifier_module, + object_name=action_generator_type_name, + ) + if not issubclass(action_generator_type, ActionGenerator): + raise NotImplementedError( + "Action generator type {} is not a subclass of the ActionGenerator base class".format( + action_generator_type.__name__ + ) + ) + return action_generator_type + + +def action_generator_specification_type( + action_generator_type: Type[ActionGenerator], +) -> Optional[Type]: + constructor_signature = inspect.signature(action_generator_type.__init__) + specification_type = None + for arg_name, arg in constructor_signature.parameters.items(): + if arg_name == "specification": + specification_type = arg.annotation + break + return specification_type diff --git a/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py b/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py index 95c756d4c0..bbbee1a469 100644 --- a/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py +++ b/monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py @@ -3,6 +3,12 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.astm.f3411 import ( DSSInstanceResource, @@ -34,6 +40,14 @@ class ForEachDSS(ActionGenerator[ForEachDSSSpecification]): _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: ForEachDSSSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: ForEachDSSSpecification, diff --git a/monitoring/uss_qualifier/action_generators/definitions.py b/monitoring/uss_qualifier/action_generators/definitions.py index 8533a5ccc1..aca51c8bfa 100644 --- a/monitoring/uss_qualifier/action_generators/definitions.py +++ b/monitoring/uss_qualifier/action_generators/definitions.py @@ -5,7 +5,7 @@ GeneratorTypeName = str -"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier.action_generators` module""" +"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier` module""" ActionGeneratorSpecificationType = TypeVar( diff --git a/monitoring/uss_qualifier/action_generators/documentation/__init__.py b/monitoring/uss_qualifier/action_generators/documentation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/action_generators/documentation/definitions.py b/monitoring/uss_qualifier/action_generators/documentation/definitions.py new file mode 100644 index 0000000000..33bb951e80 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/documentation/definitions.py @@ -0,0 +1,40 @@ +from typing import Optional + +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName +from monitoring.uss_qualifier.fileio import FileReference +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName +from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition + + +class PotentialTestScenarioAction(ImplicitDict): + scenario_type: TestScenarioTypeName + """Type of test scenario.""" + + +class PotentialTestSuiteAction(ImplicitDict): + suite_type: Optional[FileReference] + """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" + + suite_definition: Optional[TestSuiteDefinition] + """Definition of test suite internal to the configuration -- specified instead of `suite_type`.""" + + +class PotentialActionGeneratorAction(ImplicitDict): + generator_type: GeneratorTypeName + """Type of action generator.""" + + specification: dict + """Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`""" + + +class PotentialGeneratedAction(ImplicitDict): + test_scenario: Optional[PotentialTestScenarioAction] + test_suite: Optional[PotentialTestSuiteAction] + action_generator: Optional[PotentialActionGeneratorAction] + + def get_action_type(self) -> ActionType: + matches = [v for v in ActionType if v in self and self[v]] + if len(matches) != 1: + ActionType.raise_invalid_action_declaration() + return ActionType(matches[0]) diff --git a/monitoring/uss_qualifier/action_generators/documentation/documentation.py b/monitoring/uss_qualifier/action_generators/documentation/documentation.py new file mode 100644 index 0000000000..ec1193f4c5 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/documentation/documentation.py @@ -0,0 +1,78 @@ +from typing import List, Union + +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, + action_generator_specification_type, +) +from monitoring.uss_qualifier.action_generators.definitions import ( + ActionGeneratorDefinition, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, + PotentialTestSuiteAction, + PotentialActionGeneratorAction, +) +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteActionDeclaration, + ActionType, +) + + +def list_potential_actions_for_action_generator_definition( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction] +) -> List[PotentialGeneratedAction]: + action_generator_type = action_generator_type_from_name( + generator_def.generator_type + ) + specification_type = action_generator_specification_type(action_generator_type) + if specification_type is not None: + spec = ImplicitDict.parse(generator_def.specification, specification_type) + else: + spec = None + return action_generator_type.list_potential_actions(spec) + + +def list_potential_actions_for_action_declaration( + declaration: TestSuiteActionDeclaration, +) -> List[PotentialGeneratedAction]: + action_type = declaration.get_action_type() + if action_type == ActionType.TestScenario: + return [ + PotentialGeneratedAction( + test_scenario=PotentialTestScenarioAction( + scenario_type=declaration.test_scenario.scenario_type + ) + ) + ] + elif action_type == ActionType.TestSuite: + if "suite_type" in declaration.test_suite and declaration.test_suite.suite_type: + return [ + PotentialGeneratedAction( + test_suite=PotentialTestSuiteAction( + suite_type=declaration.test_suite.suite_type + ) + ) + ] + elif ( + "suite_definition" in declaration.test_suite + and declaration.test_suite.suite_definition + ): + return [ + PotentialGeneratedAction( + test_suite=PotentialTestSuiteAction( + suite_definition=declaration.test_suite.suite_definition + ) + ) + ] + elif action_type == ActionType.ActionGenerator: + return [ + PotentialGeneratedAction( + action_generator=PotentialActionGeneratorAction( + generator_type=declaration.action_generator.generator_type + ) + ) + ] + else: + raise NotImplementedError(f"Action type {action_type} is not supported") diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py index f81899c677..38d2ddf54e 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py +++ b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py @@ -3,6 +3,17 @@ from implicitdict import ImplicitDict from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, + action_generator_specification_type, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource @@ -14,7 +25,10 @@ ResourceType, ) -from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteActionDeclaration, + ActionType, +) from monitoring.uss_qualifier.suites.suite import ( ActionGenerator, TestSuiteAction, @@ -43,6 +57,14 @@ class FlightPlannerCombinations( _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: FlightPlannerCombinationsSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: FlightPlannerCombinationsSpecification, diff --git a/monitoring/uss_qualifier/action_generators/repetition/repeat.py b/monitoring/uss_qualifier/action_generators/repetition/repeat.py index c282a73693..a2f3256af2 100644 --- a/monitoring/uss_qualifier/action_generators/repetition/repeat.py +++ b/monitoring/uss_qualifier/action_generators/repetition/repeat.py @@ -1,6 +1,12 @@ from typing import Dict, List, Optional from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.resource import ResourceType @@ -26,6 +32,14 @@ class Repeat(ActionGenerator[RepeatSpecification]): _current_action: int _failure_reaction: ReactionToFailure + @classmethod + def list_potential_actions( + cls, specification: RepeatSpecification + ) -> List[PotentialGeneratedAction]: + return list_potential_actions_for_action_declaration( + specification.action_to_repeat + ) + def __init__( self, specification: RepeatSpecification, diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index d6bb83e63c..cb4371d828 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -7,6 +7,7 @@ from implicitdict import ImplicitDict, StringBasedDateTime from monitoring.monitorlib import fetch, inspection +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ( TestConfiguration, @@ -18,6 +19,7 @@ JSONPathExpression, ) from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName class FailedCheck(ImplicitDict): @@ -211,7 +213,7 @@ class TestScenarioReport(ImplicitDict): name: str """Name of this test scenario""" - scenario_type: str + scenario_type: TestScenarioTypeName """Type of this test scenario""" documentation_url: str @@ -295,7 +297,7 @@ def participant_ids(self) -> Set[ParticipantID]: class ActionGeneratorReport(ImplicitDict): - generator_type: str + generator_type: GeneratorTypeName """Type of action generator""" actions: List[TestSuiteActionReport] diff --git a/monitoring/uss_qualifier/scenarios/definitions.py b/monitoring/uss_qualifier/scenarios/definitions.py index 5672c40b10..90d8c556fc 100644 --- a/monitoring/uss_qualifier/scenarios/definitions.py +++ b/monitoring/uss_qualifier/scenarios/definitions.py @@ -5,9 +5,13 @@ from monitoring.uss_qualifier.resources.definitions import ResourceID +TestScenarioTypeName = str +"""This plain string represents a type of test scenario, expressed as a Python class name qualified relative to the `uss_qualifier` module""" + + class TestScenarioDeclaration(ImplicitDict): - scenario_type: FileReference - """Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder""" + scenario_type: TestScenarioTypeName + """Type of test scenario.""" resources: Optional[Dict[ResourceID, ResourceID]] """Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite. diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index b3901e938e..c564045ba0 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -24,7 +24,10 @@ ParticipantID, PassedCheck, ) -from monitoring.uss_qualifier.scenarios.definitions import TestScenarioDeclaration +from monitoring.uss_qualifier.scenarios.definitions import ( + TestScenarioDeclaration, + TestScenarioTypeName, +) from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestScenarioDocumentation, TestCaseDocumentation, @@ -159,7 +162,7 @@ def record_passed( self._step_report.passed_checks.append(passed_check) -def get_scenario_type_by_name(scenario_type_name: str) -> Type: +def get_scenario_type_by_name(scenario_type_name: TestScenarioTypeName) -> Type: inspection.import_submodules(scenarios_module) scenario_type = inspection.get_module_object_by_name( parent_module=uss_qualifier_module, object_name=scenario_type_name diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index b91c6ea824..ec359caacf 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -4,6 +4,7 @@ ## Actions -1. Action generator: `action_generators.astm.f3411.ForEachDSS` +1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) + 1. Suite: 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 8dd20db5b5..38be47bdb5 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -4,6 +4,7 @@ ## Actions -1. Action generator: `action_generators.astm.f3411.ForEachDSS` +1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) + 1. Suite: 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 62a6f5c093..c83a1301f3 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -4,6 +4,9 @@ ## Actions -1. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` -2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` -3. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +1. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) +2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) +3. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index ca453656d1..0b84e4bee2 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -1,18 +1,37 @@ import glob import inspect import os -from typing import Iterator, Optional +from typing import Iterator, Optional, List, Union from implicitdict import ImplicitDict +from monitoring.uss_qualifier.action_generators.action_generator import ( + action_generator_type_from_name, +) +from monitoring.uss_qualifier.action_generators.definitions import ( + ActionGeneratorDefinition, +) +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialActionGeneratorAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_generator_definition, +) from monitoring.uss_qualifier.fileio import ( load_dict_with_references, get_package_name, resolve_filename, + FileReference, ) +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name -from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition, ActionType +from monitoring.uss_qualifier.suites.definitions import ( + TestSuiteDefinition, + ActionType, + TestSuiteActionDeclaration, +) def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: @@ -45,46 +64,91 @@ def make_test_suite_documentation(test_suite_yaml_file: str) -> str: lines.append("") base_path = os.path.dirname(test_suite_yaml_file) for i, action in enumerate(suite_def.actions): - action_type = action.get_action_type() - if action_type == ActionType.TestScenario: - scenario_type = get_scenario_type_by_name( - action.test_scenario.scenario_type - ) - py_rel_path = os.path.relpath(inspect.getfile(scenario_type), base_path) - scenario_doc = get_documentation(scenario_type) - doc_rel_path = os.path.relpath(scenario_doc.local_path, start=base_path) - lines.append( - f"{i + 1}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{action.test_scenario.scenario_type}`]({py_rel_path}))" - ) - elif action_type == ActionType.TestSuite: - if "suite_type" in action.test_suite and action.test_suite.suite_type: - suite_def = ImplicitDict.parse( - load_dict_with_references(action.test_suite.suite_type), - TestSuiteDefinition, - ) - suite_path = resolve_filename(action.test_suite.suite_type) - suite_rel_path = os.path.relpath(suite_path, start=base_path) - doc_path = os.path.splitext(suite_path)[0] + ".md" - doc_rel_path = os.path.relpath(doc_path, start=base_path) - lines.append( - f"{i + 1}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{action.test_suite.suite_type}`]({suite_rel_path}))" - ) - elif "suite_definition" in action.test_suite and action.suite_definition: - # TODO: Generate additional test suite documentation for in-suite suite definition - lines.append(f"{i + 1}. Suite: ") - else: - raise ValueError( - f"Test suite action {i + 1} missing suite type or definition in {test_suite_yaml_file}" - ) - elif action_type == ActionType.ActionGenerator: - # TODO: Add documentation for action generators - lines.append( - f"{i + 1}. Action generator: `{action.action_generator.generator_type}`" - ) - else: - raise NotImplementedError( - f"Unsupported test suite action type: {action_type}" - ) + lines.extend(render_action(action, i + 1, base_path, 0)) lines.append("") return "\n".join(lines) + + +def render_scenario( + scenario_type_name: TestScenarioTypeName, i: int, base_path: str, indent: int +) -> List[str]: + lines = [] + scenario_type = get_scenario_type_by_name(scenario_type_name) + py_rel_path = os.path.relpath(inspect.getfile(scenario_type), base_path) + scenario_doc = get_documentation(scenario_type) + doc_rel_path = os.path.relpath(scenario_doc.local_path, start=base_path) + lines.append( + f"{' ' * indent}{i}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{scenario_type_name}`]({py_rel_path}))" + ) + return lines + + +def render_suite_by_type( + suite_type: FileReference, i: int, base_path: str, indent: int +) -> List[str]: + lines = [] + suite_def = ImplicitDict.parse( + load_dict_with_references(suite_type), + TestSuiteDefinition, + ) + suite_path = resolve_filename(suite_type) + suite_rel_path = os.path.relpath(suite_path, start=base_path) + doc_path = os.path.splitext(suite_path)[0] + ".md" + doc_rel_path = os.path.relpath(doc_path, start=base_path) + lines.append( + f"{' ' * indent}{i}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{suite_type}`]({suite_rel_path}))" + ) + return lines + + +def render_action_generator( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction], + i: int, + base_path: str, + indent: int, +) -> List[str]: + lines = [] + action_generator_type = action_generator_type_from_name( + generator_def.generator_type + ) + py_rel_path = os.path.relpath( + inspect.getfile(action_generator_type), start=base_path + ) + lines.append( + f"{' ' * indent}{i}. Action generator: [`{generator_def.generator_type}`]({py_rel_path})" + ) + potential_actions = list_potential_actions_for_action_generator_definition( + generator_def + ) + for j, potential_action in enumerate(potential_actions): + lines.extend(render_action(potential_action, j + 1, base_path, indent + 4)) + return lines + + +def render_action( + action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction], + i: int, + base_path: str, + indent: int, +) -> List[str]: + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + return render_scenario(action.test_scenario.scenario_type, i, base_path, indent) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + return render_suite_by_type( + action.test_suite.suite_type, i, base_path, indent + ) + elif ( + "suite_definition" in action.test_suite + and action.test_suite.suite_definition + ): + # TODO: Generate additional test suite documentation for in-suite suite definition + return [f"{' ' * indent}{i}. Suite: "] + else: + raise ValueError(f"Test suite action {i} missing suite type or definition") + elif action_type == ActionType.ActionGenerator: + return render_action_generator(action.action_generator, i, base_path, indent) + else: + raise NotImplementedError(f"Unsupported test suite action type: {action_type}") diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 25bdb21bce..a98b01af1a 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -5,4 +5,5 @@ ## Actions 1. Suite: [ASTM F3548-21](../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../astm/utm/f3548_21.yaml)) -2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Flight authorisation validation](../../scenarios/uspace/flight_auth/validation.md) ([`scenarios.uspace.flight_auth.Validation`](../../scenarios/uspace/flight_auth/validation.py)) diff --git a/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json b/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json index 60d2424977..26e32cbfaf 100644 --- a/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json +++ b/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json @@ -1,35 +1,35 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "resources": { - "type": [ - "object", - "null" - ], + "additionalProperties": { + "type": "string" + }, + "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource .", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" } }, - "additionalProperties": { - "type": "string" - }, - "description": "Mapping of the ID a resource in the test scenario -> the ID a resource is known by in the parent test suite.\n\nThe additional argument to concrete test scenario constructor is supplied by the parent suite resource ." + "type": [ + "object", + "null" + ] }, "scenario_type": { - "type": "string", - "description": "Type/location of test scenario. Usually expressed as the class name of the scenario module-qualified relative to the `uss_qualifier` folder" + "description": "Type of test scenario.", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/scenarios/definitions/TestScenarioDeclaration.json", - "description": "monitoring.uss_qualifier.scenarios.definitions.TestScenarioDeclaration, as defined in monitoring/uss_qualifier/scenarios/definitions.py", "required": [ "scenario_type" - ] + ], + "type": "object" } \ No newline at end of file From b00ffef0351436b2daa0be5b967844c0f98fc370 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 06:04:54 +0000 Subject: [PATCH 2/8] Generate documentation for in-suite suite definitions --- .../suites/astm/netrid/f3411_19.md | 2 +- .../suites/astm/netrid/f3411_19_suite1.md | 7 ++ .../suites/astm/netrid/f3411_22a.md | 2 +- .../suites/astm/netrid/f3411_22a_suite1.md | 8 ++ .../suites/documentation/documentation.py | 119 +++++++++++++++--- .../documentation/format_documentation.py | 17 ++- 6 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md create mode 100644 monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index ec359caacf..14feae21b7 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -5,6 +5,6 @@ ## Actions 1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) - 1. Suite: + 1. Suite: [DSS instance validator](f3411_19_suite1.md) ([in-suite definition](f3411_19.yaml)) 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md new file mode 100644 index 0000000000..ddae48161b --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md @@ -0,0 +1,7 @@ + +# DSS instance validator test suite +Defined in [parent suite](f3411_19.md) [`suites.astm.netrid.f3411_19`](./f3411_19.yaml) + +## Actions + +1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 38be47bdb5..64b779f6dd 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -5,6 +5,6 @@ ## Actions 1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) - 1. Suite: + 1. Suite: [DSS instance validator](f3411_22a_suite1.md) ([in-suite definition](f3411_22a.yaml)) 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md new file mode 100644 index 0000000000..7d426750b0 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md @@ -0,0 +1,8 @@ + +# DSS instance validator test suite +Defined in [parent suite](f3411_22a.md) [`suites.astm.netrid.f3411_22a`](./f3411_22a.yaml) + +## Actions + +1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) +2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index 0b84e4bee2..af27db8f81 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -1,7 +1,7 @@ import glob import inspect import os -from typing import Iterator, Optional, List, Union +from typing import Iterator, Optional, List, Union, Dict from implicitdict import ImplicitDict from monitoring.uss_qualifier.action_generators.action_generator import ( @@ -47,30 +47,52 @@ def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: yield suite -def make_test_suite_documentation(test_suite_yaml_file: str) -> str: +def make_test_suite_documentation( + suite_def: TestSuiteDefinition, + suite_yaml_file: str, + suite_doc_file: str, + parent_suite_doc: Optional[str] = None, +) -> Dict[str, str]: + test_suites: Dict[str, str] = {} + lines = [] - suite_def: TestSuiteDefinition = ImplicitDict.parse( - load_dict_with_references("file://" + test_suite_yaml_file), TestSuiteDefinition - ) lines.append( "" ) lines.append(f"# {suite_def.name} test suite") - local_path = os.path.split(test_suite_yaml_file)[-1] - lines.append(f"[`{get_package_name(test_suite_yaml_file)}`](./{local_path})") + local_path = os.path.split(suite_yaml_file)[-1] + if parent_suite_doc is None: + prefix = "" + else: + parent_rel_path = os.path.relpath( + parent_suite_doc, start=os.path.dirname(suite_doc_file) + ) + prefix = f"Defined in [parent suite]({parent_rel_path}) " + lines.append(f"{prefix}[`{get_package_name(suite_yaml_file)}`](./{local_path})") lines.append("") lines.append("## Actions") lines.append("") - base_path = os.path.dirname(test_suite_yaml_file) + base_path = os.path.dirname(suite_yaml_file) for i, action in enumerate(suite_def.actions): - lines.extend(render_action(action, i + 1, base_path, 0)) + lines.extend( + _render_action( + action, + i + 1, + base_path, + 0, + suite_yaml_file, + suite_doc_file, + test_suites, + ) + ) lines.append("") - return "\n".join(lines) + test_suites[suite_doc_file] = "\n".join(lines) + return test_suites -def render_scenario( +def _render_scenario( scenario_type_name: TestScenarioTypeName, i: int, base_path: str, indent: int ) -> List[str]: lines = [] @@ -84,7 +106,7 @@ def render_scenario( return lines -def render_suite_by_type( +def _render_suite_by_type( suite_type: FileReference, i: int, base_path: str, indent: int ) -> List[str]: lines = [] @@ -102,11 +124,38 @@ def render_suite_by_type( return lines -def render_action_generator( +def _render_suite_by_definition( + suite_def: TestSuiteDefinition, + i: int, + base_path: str, + indent: int, + parent_yaml_file: str, + parent_doc_file: str, + test_suites: Dict[str, str], +) -> List[str]: + doc_path = os.path.splitext(parent_doc_file)[0] + f"_suite{i}.md" + new_docs = make_test_suite_documentation( + suite_def, parent_yaml_file, doc_path, parent_doc_file + ) + + for k, v in new_docs.items(): + test_suites[k] = v + + doc_rel_path = os.path.relpath(doc_path, base_path) + parent_rel_path = os.path.relpath(parent_yaml_file, start=base_path) + return [ + f"{' ' * indent}{i}. Suite: [{suite_def.name}]({doc_rel_path}) ([in-suite definition]({parent_rel_path}))" + ] + + +def _render_action_generator( generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction], i: int, base_path: str, indent: int, + parent_yaml_file: str, + parent_doc_file: str, + test_suites: Dict[str, str], ) -> List[str]: lines = [] action_generator_type = action_generator_type_from_name( @@ -122,33 +171,63 @@ def render_action_generator( generator_def ) for j, potential_action in enumerate(potential_actions): - lines.extend(render_action(potential_action, j + 1, base_path, indent + 4)) + lines.extend( + _render_action( + potential_action, + j + 1, + base_path, + indent + 4, + parent_yaml_file, + parent_doc_file, + test_suites, + ) + ) return lines -def render_action( +def _render_action( action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction], i: int, base_path: str, indent: int, + parent_yaml_file: str, + parent_doc_file: str, + test_suites: Dict[str, str], ) -> List[str]: action_type = action.get_action_type() if action_type == ActionType.TestScenario: - return render_scenario(action.test_scenario.scenario_type, i, base_path, indent) + return _render_scenario( + action.test_scenario.scenario_type, i, base_path, indent + ) elif action_type == ActionType.TestSuite: if "suite_type" in action.test_suite and action.test_suite.suite_type: - return render_suite_by_type( + return _render_suite_by_type( action.test_suite.suite_type, i, base_path, indent ) elif ( "suite_definition" in action.test_suite and action.test_suite.suite_definition ): - # TODO: Generate additional test suite documentation for in-suite suite definition - return [f"{' ' * indent}{i}. Suite: "] + return _render_suite_by_definition( + action.test_suite.suite_definition, + i, + base_path, + indent, + parent_yaml_file, + parent_doc_file, + test_suites, + ) else: raise ValueError(f"Test suite action {i} missing suite type or definition") elif action_type == ActionType.ActionGenerator: - return render_action_generator(action.action_generator, i, base_path, indent) + return _render_action_generator( + action.action_generator, + i, + base_path, + indent, + parent_yaml_file, + parent_doc_file, + test_suites, + ) else: raise NotImplementedError(f"Unsupported test suite action type: {action_type}") diff --git a/monitoring/uss_qualifier/suites/documentation/format_documentation.py b/monitoring/uss_qualifier/suites/documentation/format_documentation.py index 10b3ea2a5a..e04d5038d6 100644 --- a/monitoring/uss_qualifier/suites/documentation/format_documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/format_documentation.py @@ -2,6 +2,9 @@ import os import sys +from implicitdict import ImplicitDict +from monitoring.uss_qualifier.fileio import load_dict_with_references +from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition from monitoring.uss_qualifier.suites.documentation.documentation import ( find_test_suites, make_test_suite_documentation, @@ -9,10 +12,20 @@ def main(lint: bool) -> int: - changes = False + test_suite_docs = {} for suite_yaml_file in find_test_suites(): - suite_doc_content = make_test_suite_documentation(suite_yaml_file) + suite_def: TestSuiteDefinition = ImplicitDict.parse( + load_dict_with_references("file://" + suite_yaml_file), TestSuiteDefinition + ) suite_doc_file = os.path.splitext(suite_yaml_file)[0] + ".md" + new_docs = make_test_suite_documentation( + suite_def, suite_yaml_file, suite_doc_file + ) + for k, v in new_docs.items(): + test_suite_docs[k] = v + + changes = False + for suite_doc_file, suite_doc_content in test_suite_docs.items(): if os.path.exists(suite_doc_file): with open(suite_doc_file, "r") as f: existing_content = f.read() From 9dbd584f24c268b020b924a77e142a20fe370536 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 06:38:08 +0000 Subject: [PATCH 3/8] Remove orphaned documentation --- .../uss_qualifier/suites/interuss/generate_test_data.md | 8 -------- .../suites/interuss/generate_test_data_twice.md | 7 ------- 2 files changed, 15 deletions(-) delete mode 100644 monitoring/uss_qualifier/suites/interuss/generate_test_data.md delete mode 100644 monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md diff --git a/monitoring/uss_qualifier/suites/interuss/generate_test_data.md b/monitoring/uss_qualifier/suites/interuss/generate_test_data.md deleted file mode 100644 index 662ddf6765..0000000000 --- a/monitoring/uss_qualifier/suites/interuss/generate_test_data.md +++ /dev/null @@ -1,8 +0,0 @@ - -# Generate RID test data test suite -[`suites.interuss.generate_test_data`](./generate_test_data.yaml) - -## Actions - -1. Scenario: [Store NetRID flight data](../../scenarios/astm/netrid/store_flight_data.md) ([`scenarios.astm.netrid.StoreFlightData`](../../scenarios/astm/netrid/store_flight_data.py)) -2. Scenario: [Store NetRID flight data](../../scenarios/astm/netrid/store_flight_data.md) ([`scenarios.astm.netrid.StoreFlightData`](../../scenarios/astm/netrid/store_flight_data.py)) diff --git a/monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md b/monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md deleted file mode 100644 index 540abc5a0e..0000000000 --- a/monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md +++ /dev/null @@ -1,7 +0,0 @@ - -# Generate RID test data twice test suite -[`suites.interuss.generate_test_data_twice`](./generate_test_data_twice.yaml) - -## Actions - -1. Action generator: `action_generators.Repeat` From 8474c979987c697e8bb15098677f41ec3f5d9312 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 15:25:07 +0000 Subject: [PATCH 4/8] Replace long arg lists with context --- .../suites/documentation/documentation.py | 125 ++++++++---------- 1 file changed, 55 insertions(+), 70 deletions(-) diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index af27db8f81..9a02b86185 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -1,6 +1,7 @@ import glob import inspect import os +from dataclasses import dataclass from typing import Iterator, Optional, List, Union, Dict from implicitdict import ImplicitDict @@ -34,6 +35,16 @@ ) +@dataclass +class TestSuiteRenderContext(object): + parent_yaml_file: str + parent_doc_file: str + base_path: str + list_index: int + indent: int + test_suites: Dict[str, str] + + def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: if start_path is None: start_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) @@ -78,12 +89,14 @@ def make_test_suite_documentation( lines.extend( _render_action( action, - i + 1, - base_path, - 0, - suite_yaml_file, - suite_doc_file, - test_suites, + TestSuiteRenderContext( + parent_yaml_file=suite_yaml_file, + parent_doc_file=suite_doc_file, + base_path=base_path, + list_index=i + 1, + indent=0, + test_suites=test_suites, + ), ) ) @@ -93,21 +106,21 @@ def make_test_suite_documentation( def _render_scenario( - scenario_type_name: TestScenarioTypeName, i: int, base_path: str, indent: int + scenario_type_name: TestScenarioTypeName, context: TestSuiteRenderContext ) -> List[str]: lines = [] scenario_type = get_scenario_type_by_name(scenario_type_name) - py_rel_path = os.path.relpath(inspect.getfile(scenario_type), base_path) + py_rel_path = os.path.relpath(inspect.getfile(scenario_type), context.base_path) scenario_doc = get_documentation(scenario_type) - doc_rel_path = os.path.relpath(scenario_doc.local_path, start=base_path) + doc_rel_path = os.path.relpath(scenario_doc.local_path, start=context.base_path) lines.append( - f"{' ' * indent}{i}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{scenario_type_name}`]({py_rel_path}))" + f"{' ' * context.indent}{context.list_index}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{scenario_type_name}`]({py_rel_path}))" ) return lines def _render_suite_by_type( - suite_type: FileReference, i: int, base_path: str, indent: int + suite_type: FileReference, context: TestSuiteRenderContext ) -> List[str]: lines = [] suite_def = ImplicitDict.parse( @@ -115,57 +128,48 @@ def _render_suite_by_type( TestSuiteDefinition, ) suite_path = resolve_filename(suite_type) - suite_rel_path = os.path.relpath(suite_path, start=base_path) + suite_rel_path = os.path.relpath(suite_path, start=context.base_path) doc_path = os.path.splitext(suite_path)[0] + ".md" - doc_rel_path = os.path.relpath(doc_path, start=base_path) + doc_rel_path = os.path.relpath(doc_path, start=context.base_path) lines.append( - f"{' ' * indent}{i}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{suite_type}`]({suite_rel_path}))" + f"{' ' * context.indent}{context.list_index}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{suite_type}`]({suite_rel_path}))" ) return lines def _render_suite_by_definition( - suite_def: TestSuiteDefinition, - i: int, - base_path: str, - indent: int, - parent_yaml_file: str, - parent_doc_file: str, - test_suites: Dict[str, str], + suite_def: TestSuiteDefinition, context: TestSuiteRenderContext ) -> List[str]: - doc_path = os.path.splitext(parent_doc_file)[0] + f"_suite{i}.md" + doc_path = ( + os.path.splitext(context.parent_doc_file)[0] + f"_suite{context.list_index}.md" + ) new_docs = make_test_suite_documentation( - suite_def, parent_yaml_file, doc_path, parent_doc_file + suite_def, context.parent_yaml_file, doc_path, context.parent_doc_file ) for k, v in new_docs.items(): - test_suites[k] = v + context.test_suites[k] = v - doc_rel_path = os.path.relpath(doc_path, base_path) - parent_rel_path = os.path.relpath(parent_yaml_file, start=base_path) + doc_rel_path = os.path.relpath(doc_path, context.base_path) + parent_rel_path = os.path.relpath(context.parent_yaml_file, start=context.base_path) return [ - f"{' ' * indent}{i}. Suite: [{suite_def.name}]({doc_rel_path}) ([in-suite definition]({parent_rel_path}))" + f"{' ' * context.indent}{context.list_index}. Suite: [{suite_def.name}]({doc_rel_path}) ([in-suite definition]({parent_rel_path}))" ] def _render_action_generator( generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction], - i: int, - base_path: str, - indent: int, - parent_yaml_file: str, - parent_doc_file: str, - test_suites: Dict[str, str], + context: TestSuiteRenderContext, ) -> List[str]: lines = [] action_generator_type = action_generator_type_from_name( generator_def.generator_type ) py_rel_path = os.path.relpath( - inspect.getfile(action_generator_type), start=base_path + inspect.getfile(action_generator_type), start=context.base_path ) lines.append( - f"{' ' * indent}{i}. Action generator: [`{generator_def.generator_type}`]({py_rel_path})" + f"{' ' * context.indent}{context.list_index}. Action generator: [`{generator_def.generator_type}`]({py_rel_path})" ) potential_actions = list_potential_actions_for_action_generator_definition( generator_def @@ -174,12 +178,14 @@ def _render_action_generator( lines.extend( _render_action( potential_action, - j + 1, - base_path, - indent + 4, - parent_yaml_file, - parent_doc_file, - test_suites, + TestSuiteRenderContext( + parent_yaml_file=context.parent_yaml_file, + parent_doc_file=context.parent_doc_file, + base_path=context.base_path, + list_index=j + 1, + indent=context.indent + 4, + test_suites=context.test_suites, + ), ) ) return lines @@ -187,47 +193,26 @@ def _render_action_generator( def _render_action( action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction], - i: int, - base_path: str, - indent: int, - parent_yaml_file: str, - parent_doc_file: str, - test_suites: Dict[str, str], + context: TestSuiteRenderContext, ) -> List[str]: action_type = action.get_action_type() if action_type == ActionType.TestScenario: - return _render_scenario( - action.test_scenario.scenario_type, i, base_path, indent - ) + return _render_scenario(action.test_scenario.scenario_type, context) elif action_type == ActionType.TestSuite: if "suite_type" in action.test_suite and action.test_suite.suite_type: - return _render_suite_by_type( - action.test_suite.suite_type, i, base_path, indent - ) + return _render_suite_by_type(action.test_suite.suite_type, context) elif ( "suite_definition" in action.test_suite and action.test_suite.suite_definition ): return _render_suite_by_definition( - action.test_suite.suite_definition, - i, - base_path, - indent, - parent_yaml_file, - parent_doc_file, - test_suites, + action.test_suite.suite_definition, context ) else: - raise ValueError(f"Test suite action {i} missing suite type or definition") + raise ValueError( + f"Test suite action {context.list_index} missing suite type or definition" + ) elif action_type == ActionType.ActionGenerator: - return _render_action_generator( - action.action_generator, - i, - base_path, - indent, - parent_yaml_file, - parent_doc_file, - test_suites, - ) + return _render_action_generator(action.action_generator, context) else: raise NotImplementedError(f"Unsupported test suite action type: {action_type}") From be9cfb0c20856e0a7f43266a8fc03a02aa640312 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 16:13:19 +0000 Subject: [PATCH 5/8] Add checked requirements --- .../scenarios/documentation/parsing.py | 8 +- .../suites/astm/netrid/f3411_19.md | 198 +++++++++++ .../suites/astm/netrid/f3411_19_suite1.md | 23 ++ .../suites/astm/netrid/f3411_22a.md | 225 +++++++++++++ .../suites/astm/netrid/f3411_22a_suite1.md | 27 ++ .../uss_qualifier/suites/astm/utm/f3548_21.md | 96 ++++++ .../suites/documentation/documentation.py | 165 +++++++++- .../suites/faa/uft/message_signing.md | 96 ++++++ .../suites/uspace/flight_auth.md | 96 ++++++ .../suites/uspace/geo_awareness_cis.md | 4 + .../suites/uspace/network_identification.md | 225 +++++++++++++ .../suites/uspace/required_services.md | 311 ++++++++++++++++++ 12 files changed, 1471 insertions(+), 3 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index cdec8572ba..377be9c5d0 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -10,6 +10,8 @@ from monitoring import uss_qualifier as uss_qualifier_module from monitoring.monitorlib import versioning from monitoring.monitorlib.inspection import fullname, get_module_object_by_name +from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestStepDocumentation, TestCheckDocumentation, @@ -80,7 +82,7 @@ def _parse_test_check( if isinstance(values[c], marko.block.Paragraph): for child in values[c].children: if isinstance(child, marko.inline.StrongEmphasis): - reqs.append(_text_of(child)) + reqs.append(RequirementID(_text_of(child))) c += 1 return TestCheckDocumentation(name=name, url=url, applicable_requirements=reqs) @@ -310,6 +312,8 @@ def get_documentation(scenario: Type) -> TestScenarioDocumentation: return getattr(scenario, DOC_CACHE_ATTRIBUTE) -def get_documentation_by_name(scenario_type_name: str) -> TestScenarioDocumentation: +def get_documentation_by_name( + scenario_type_name: TestScenarioTypeName, +) -> TestScenarioDocumentation: scenario_type = get_module_object_by_name(uss_qualifier_module, scenario_type_name) return get_documentation(scenario_type) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 14feae21b7..067f52abb1 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -8,3 +8,201 @@ 1. Suite: [DSS instance validator](f3411_19_suite1.md) ([in-suite definition](f3411_19.yaml)) 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3411.v19A2-6-1,1aASTM F3411-19 NetRID DSS interoperability
A2-6-1,1bASTM F3411-19 NetRID DSS interoperability
A2-6-1,1cASTM F3411-19 NetRID DSS interoperability
A2-6-1,1dASTM F3411-19 NetRID DSS interoperability
A2-6-1,2aASTM F3411-19 NetRID DSS interoperability
A2-6-1,2bASTM F3411-19 NetRID DSS interoperability
A2-6-1,3aASTM F3411-19 NetRID DSS interoperability
A2-6-1,3bASTM F3411-19 NetRID DSS interoperability
A2-6-1,3cASTM F3411-19 NetRID DSS interoperability
A2-6-1,3dASTM F3411-19 NetRID DSS interoperability
A2-6-1,4aASTM F3411-19 NetRID DSS interoperability
A2-6-1,4bASTM F3411-19 NetRID DSS interoperability
A2-6-1,5ASTM F3411-19 NetRID DSS interoperability
A2-6-1,6ASTM F3411-19 NetRID DSS interoperability
DSS0030ASTM NetRID nominal behavior
DSS0070ASTM F3411-19 NetRID DSS interoperability
DSS0130ASTM F3411-19 NetRID DSS interoperability
DSS0130,2,aASTM F3411-19 NetRID DSS interoperability
DSS0130,2,bASTM F3411-19 NetRID DSS interoperability
DSS0130,2,cASTM F3411-19 NetRID DSS interoperability
DSS0130,2,dASTM F3411-19 NetRID DSS interoperability
DSS0130,2,fASTM F3411-19 NetRID DSS interoperability
DSS0130,3,aASTM F3411-19 NetRID DSS interoperability
DSS0130,3,bASTM F3411-19 NetRID DSS interoperability
DSS0130,3,cASTM F3411-19 NetRID DSS interoperability
DSS0130,3,dASTM F3411-19 NetRID DSS interoperability
DSS0210ASTM F3411-19 NetRID DSS interoperability
NET0260ASTM NetRID nominal behavior
NET0270ASTM NetRID nominal behavior
NET0290ASTM NetRID nominal behavior
NET0430ASTM NetRID nominal behavior
NET0470ASTM NetRID nominal behavior
NET0480ASTM NetRID nominal behavior
NET0490ASTM NetRID nominal behavior
NET0500ASTM NetRID nominal behavior
NET0610ASTM NetRID nominal behavior
NET0710ASTM NetRID nominal behavior
astm.f3411.v22aDSS0030ASTM NetRID DSS: Subscription Validation
DSS0050ASTM NetRID DSS: Subscription Validation
DSS0060ASTM NetRID DSS: Subscription Validation
interuss.automated_testing.rid.injectionDeleteTestSuccessASTM NetRID nominal behavior
ExpectedBehaviorASTM NetRID nominal behavior
UpsertTestResultASTM NetRID nominal behavior
UpsertTestSuccessASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessASTM NetRID nominal behavior
UniqueFlightsASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md index ddae48161b..f5ce9031c2 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md @@ -5,3 +5,26 @@ Defined in [parent suite](f3411_19.md) [`suites.astm.netrid.f3411_19`](./f3411_1 ## Actions 1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3411.v22aDSS0030ASTM NetRID DSS: Subscription Validation
DSS0050ASTM NetRID DSS: Subscription Validation
DSS0060ASTM NetRID DSS: Subscription Validation
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 64b779f6dd..5a8e2e8d20 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -8,3 +8,228 @@ 1. Suite: [DSS instance validator](f3411_22a_suite1.md) ([in-suite definition](f3411_22a.yaml)) 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3411.v22aA2-6-1,1aASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ASTM F3411-22a NetRID DSS interoperability
DSS0030ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ASTM NetRID DSS: Subscription Validation
DSS0060ASTM NetRID DSS: Subscription Validation
DSS0070ASTM F3411-22a NetRID DSS interoperability
DSS0130ASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dASTM F3411-22a NetRID DSS interoperability
DSS0210ASTM F3411-22a NetRID DSS interoperability
NET0260ASTM NetRID nominal behavior
NET0260,Table1,1ASTM NetRID nominal behavior
NET0260,Table1,1aASTM NetRID nominal behavior
NET0260,Table1,23ASTM NetRID nominal behavior
NET0260,Table1,24ASTM NetRID nominal behavior
NET0260,Table1,25ASTM NetRID nominal behavior
NET0260,Table1,26ASTM NetRID nominal behavior
NET0260,Table1,7ASTM NetRID nominal behavior
NET0260,Table1,9ASTM NetRID nominal behavior
NET0270ASTM NetRID nominal behavior
NET0290ASTM NetRID nominal behavior
NET0430ASTM NetRID nominal behavior
NET0470ASTM NetRID nominal behavior
NET0480ASTM NetRID nominal behavior
NET0490ASTM NetRID nominal behavior
NET0500ASTM NetRID nominal behavior
NET0610ASTM NetRID nominal behavior
NET0710ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
interuss.automated_testing.rid.injectionDeleteTestSuccessASTM NetRID nominal behavior
ExpectedBehaviorASTM NetRID nominal behavior
UpsertTestResultASTM NetRID nominal behavior
UpsertTestSuccessASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessASTM NetRID nominal behavior
UniqueFlightsASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md index 7d426750b0..222e6145e6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md @@ -6,3 +6,30 @@ Defined in [parent suite](f3411_22a.md) [`suites.astm.netrid.f3411_22a`](./f3411 1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) 2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3411.v22aDSS0030ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
DSS0050ASTM NetRID DSS: Subscription Validation
DSS0060ASTM NetRID DSS: Subscription Validation
NET0710ASTM NetRID DSS: Simple ISA
diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index c83a1301f3..8b595b012e 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -10,3 +10,99 @@ 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) 3. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3548.v21DSS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0310Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0500Validation of operational intents
OPIN0015Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0020Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0025Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0030Validation of operational intents
OPIN0040Validation of operational intents
SCD0015Nominal planning: conflict with higher priority
SCD0020Nominal planning: conflict with higher priority
SCD0025Nominal planning: conflict with higher priority
SCD0030Nominal planning: conflict with higher priority
SCD0035Nominal planning: not permitted conflict with equal priority
SCD0040Nominal planning: not permitted conflict with equal priority
SCD0045Nominal planning: not permitted conflict with equal priority
SCD0050Nominal planning: not permitted conflict with equal priority
USS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
USS0105Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
interuss.automated_testing.flight_planningClearAreaValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
DeleteFlightSuccessValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
ExpectedBehaviorValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index 9a02b86185..e0bdbdef7f 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -1,3 +1,4 @@ +from __future__ import annotations import glob import inspect import os @@ -25,14 +26,24 @@ resolve_filename, FileReference, ) +from monitoring.uss_qualifier.requirements.definitions import RequirementID +from monitoring.uss_qualifier.requirements.documentation import get_requirement from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName -from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation +from monitoring.uss_qualifier.scenarios.documentation.definitions import ( + TestScenarioDocumentation, + TestCheckDocumentation, +) +from monitoring.uss_qualifier.scenarios.documentation.parsing import ( + get_documentation, + get_documentation_by_name, +) from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name from monitoring.uss_qualifier.suites.definitions import ( TestSuiteDefinition, ActionType, TestSuiteActionDeclaration, ) +from monitoring.uss_qualifier.suites.suite import TestSuiteAction @dataclass @@ -99,8 +110,58 @@ def make_test_suite_documentation( ), ) ) + lines.append("") + lines.append("## Checked requirements") lines.append("") + reqs = _collect_requirements_from_suite_def(suite_def) + if not reqs: + lines.append( + "_This test suite documentation does not indicate that any requirements are checked._" + ) + else: + # Use an HTML table rather than Markdown table to enabled advanced features like spans + lines.append("") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + + req_ids_by_package: Dict[str, List[RequirementID]] = {} + for req_id, req in reqs.items(): + package = req_ids_by_package.get(req_id.package(), []) + if req_id not in package: + package.append(req_id) + req_ids_by_package[req_id.package()] = package + + for package in sorted(req_ids_by_package): + req_md_path = os.path.relpath( + req_ids_by_package[package][0].md_file_path(), start=base_path + ) + package_line = f' ' + for req_id in sorted(req_ids_by_package[package]): + req_text = f'{req_id.requirement_name()}' + + checked_in = list( + set( + f'{c.scenario.name}' + for c in reqs[req_id].checked_in + ) + ) + checked_in.sort() + checked_in_text = f"{'
'.join(checked_in)}" + + lines.append(" ") + if package_line: + lines.append(package_line) + package_line = None + lines.append(f" ") + lines.append(f" ") + lines.append(" ") + lines.append("
PackageRequirementChecked in
{package}
{req_text}{checked_in_text}
") + lines.append("") + test_suites[suite_doc_file] = "\n".join(lines) return test_suites @@ -216,3 +277,105 @@ def _render_action( return _render_action_generator(action.action_generator, context) else: raise NotImplementedError(f"Unsupported test suite action type: {action_type}") + + +@dataclass +class SuiteLocation(object): + scenario: TestScenarioDocumentation + check: TestCheckDocumentation + + +@dataclass +class RequirementInSuite(object): + checked_in: List[SuiteLocation] + + def extend(self, other: RequirementInSuite): + self.checked_in.extend(other.checked_in) + + +def _collect_requirements_from_suite_def( + suite_def: TestSuiteDefinition, +) -> Dict[RequirementID, RequirementInSuite]: + reqs: Dict[RequirementID, RequirementInSuite] = {} + for action in suite_def.actions: + new_reqs = _collect_requirements_from_action(action) + for req_id, req in new_reqs.items(): + if req_id not in reqs: + reqs[req_id] = req + else: + reqs[req_id].extend(req) + return reqs + + +def _collect_requirements_from_action( + action: Union[TestSuiteActionDeclaration, PotentialGeneratedAction] +) -> Dict[RequirementID, RequirementInSuite]: + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + return _collect_requirements_from_scenario(action.test_scenario.scenario_type) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + suite_def = ImplicitDict.parse( + load_dict_with_references(action.test_suite.suite_type), + TestSuiteDefinition, + ) + return _collect_requirements_from_suite_def(suite_def) + elif ( + "suite_definition" in action.test_suite + and action.test_suite.suite_definition + ): + return _collect_requirements_from_suite_def( + action.test_suite.suite_definition + ) + else: + raise ValueError( + "Neither suite_type nor suite_definition specified in test_suite action" + ) + elif action_type == ActionType.ActionGenerator: + return _collect_requirements_from_action_generator(action.action_generator) + else: + raise NotImplementedError( + f"Test suite action type {action_type} not yet supported" + ) + + +def _collect_requirements_from_scenario( + scenario_type: TestScenarioTypeName, +) -> Dict[RequirementID, RequirementInSuite]: + docs = get_documentation_by_name(scenario_type) + reqs: Dict[RequirementID, RequirementInSuite] = {} + + def add_req(req_id: RequirementID, check: TestCheckDocumentation) -> None: + req = reqs.get(req_id, RequirementInSuite(checked_in=[])) + req.checked_in.append(SuiteLocation(scenario=docs, check=check)) + reqs[req_id] = req + + for case in docs.cases: + for step in case.steps: + for check in step.checks: + for req_id in check.applicable_requirements: + add_req(req_id, check) + if "cleanup" in docs and docs.cleanup: + for check in docs.cleanup.checks: + for req_id in check.applicable_requirements: + add_req(req_id, check) + return reqs + + +def _collect_requirements_from_action_generator( + generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction] +) -> Dict[RequirementID, RequirementInSuite]: + potential_actions = list_potential_actions_for_action_generator_definition( + generator_def + ) + + reqs: Dict[RequirementID, RequirementInSuite] = {} + for potential_action in potential_actions: + new_reqs = _collect_requirements_from_action(potential_action) + for req_id, req in new_reqs.items(): + if req_id not in reqs: + reqs[req_id] = req + else: + reqs[req_id].extend(req) + + return reqs diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index a5fe6c401d..c0e5999279 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -7,3 +7,99 @@ 1. Scenario: [Start message signing](../../../scenarios/faa/uft/message_signing_start.md) ([`scenarios.faa.uft.StartMessageSigningReport`](../../../scenarios/faa/uft/message_signing_start.py)) 2. Suite: [ASTM F3548-21](../../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../../astm/utm/f3548_21.yaml)) 3. Scenario: [Finalize message signing](../../../scenarios/faa/uft/message_signing_finalize.md) ([`scenarios.faa.uft.FinalizeMessageSigningReport`](../../../scenarios/faa/uft/message_signing_finalize.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3548.v21DSS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0310Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0500Validation of operational intents
OPIN0015Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0020Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0025Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0030Validation of operational intents
OPIN0040Validation of operational intents
SCD0015Nominal planning: conflict with higher priority
SCD0020Nominal planning: conflict with higher priority
SCD0025Nominal planning: conflict with higher priority
SCD0030Nominal planning: conflict with higher priority
SCD0035Nominal planning: not permitted conflict with equal priority
SCD0040Nominal planning: not permitted conflict with equal priority
SCD0045Nominal planning: not permitted conflict with equal priority
SCD0050Nominal planning: not permitted conflict with equal priority
USS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
USS0105Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
interuss.automated_testing.flight_planningClearAreaValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
DeleteFlightSuccessValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
ExpectedBehaviorValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index a98b01af1a..735fc86ea7 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -7,3 +7,99 @@ 1. Suite: [ASTM F3548-21](../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../astm/utm/f3548_21.yaml)) 2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Flight authorisation validation](../../scenarios/uspace/flight_auth/validation.md) ([`scenarios.uspace.flight_auth.Validation`](../../scenarios/uspace/flight_auth/validation.py)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3548.v21DSS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0310Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0500Validation of operational intents
OPIN0015Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0020Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0025Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0030Validation of operational intents
OPIN0040Validation of operational intents
SCD0015Nominal planning: conflict with higher priority
SCD0020Nominal planning: conflict with higher priority
SCD0025Nominal planning: conflict with higher priority
SCD0030Nominal planning: conflict with higher priority
SCD0035Nominal planning: not permitted conflict with equal priority
SCD0040Nominal planning: not permitted conflict with equal priority
SCD0045Nominal planning: not permitted conflict with equal priority
SCD0050Nominal planning: not permitted conflict with equal priority
USS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
USS0105Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
interuss.automated_testing.flight_planningClearAreaValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
DeleteFlightSuccessValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation
ExpectedBehaviorValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation
diff --git a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md index 910b3700fb..22d7140529 100644 --- a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md +++ b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md @@ -5,3 +5,7 @@ ## Actions 1. Scenario: [EUROCAE ED-269 UAS geographical zone model](../../scenarios/eurocae/ed269/source_data_model.md) ([`scenarios.eurocae.ed269.source_data_model.SourceDataModelValidation`](../../scenarios/eurocae/ed269/source_data_model.py)) + +## Checked requirements + +_This test suite documentation does not indicate that any requirements are checked._ diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 1cfec80a32..26c7405cf8 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -5,3 +5,228 @@ ## Actions 1. Suite: [ASTM F3411-22a](../astm/netrid/f3411_22a.md) ([`suites.astm.netrid.f3411_22a`](../astm/netrid/f3411_22a.yaml)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3411.v22aA2-6-1,1aASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ASTM F3411-22a NetRID DSS interoperability
DSS0030ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ASTM NetRID DSS: Subscription Validation
DSS0060ASTM NetRID DSS: Subscription Validation
DSS0070ASTM F3411-22a NetRID DSS interoperability
DSS0130ASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dASTM F3411-22a NetRID DSS interoperability
DSS0210ASTM F3411-22a NetRID DSS interoperability
NET0260ASTM NetRID nominal behavior
NET0260,Table1,1ASTM NetRID nominal behavior
NET0260,Table1,1aASTM NetRID nominal behavior
NET0260,Table1,23ASTM NetRID nominal behavior
NET0260,Table1,24ASTM NetRID nominal behavior
NET0260,Table1,25ASTM NetRID nominal behavior
NET0260,Table1,26ASTM NetRID nominal behavior
NET0260,Table1,7ASTM NetRID nominal behavior
NET0260,Table1,9ASTM NetRID nominal behavior
NET0270ASTM NetRID nominal behavior
NET0290ASTM NetRID nominal behavior
NET0430ASTM NetRID nominal behavior
NET0470ASTM NetRID nominal behavior
NET0480ASTM NetRID nominal behavior
NET0490ASTM NetRID nominal behavior
NET0500ASTM NetRID nominal behavior
NET0610ASTM NetRID nominal behavior
NET0710ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
interuss.automated_testing.rid.injectionDeleteTestSuccessASTM NetRID nominal behavior
ExpectedBehaviorASTM NetRID nominal behavior
UpsertTestResultASTM NetRID nominal behavior
UpsertTestSuccessASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessASTM NetRID nominal behavior
UniqueFlightsASTM NetRID nominal behavior
diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 6598ea51be..f96aa06249 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -6,3 +6,314 @@ 1. Suite: [U-space flight authorisation](flight_auth.md) ([`suites.uspace.flight_auth`](flight_auth.yaml)) 2. Suite: [U-Space network identification](network_identification.md) ([`suites.uspace.network_identification`](network_identification.yaml)) + +## Checked requirements + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageRequirementChecked in
astm.f3411.v22aA2-6-1,1aASTM F3411-22a NetRID DSS interoperability
A2-6-1,1bASTM F3411-22a NetRID DSS interoperability
A2-6-1,1cASTM F3411-22a NetRID DSS interoperability
A2-6-1,1dASTM F3411-22a NetRID DSS interoperability
A2-6-1,2aASTM F3411-22a NetRID DSS interoperability
A2-6-1,2bASTM F3411-22a NetRID DSS interoperability
A2-6-1,3aASTM F3411-22a NetRID DSS interoperability
A2-6-1,3bASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dASTM F3411-22a NetRID DSS interoperability
A2-6-1,4aASTM F3411-22a NetRID DSS interoperability
A2-6-1,4bASTM F3411-22a NetRID DSS interoperability
A2-6-1,5ASTM F3411-22a NetRID DSS interoperability
A2-6-1,6ASTM F3411-22a NetRID DSS interoperability
DSS0030ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior
DSS0050ASTM NetRID DSS: Subscription Validation
DSS0060ASTM NetRID DSS: Subscription Validation
DSS0070ASTM F3411-22a NetRID DSS interoperability
DSS0130ASTM F3411-22a NetRID DSS interoperability
DSS0130,2,aASTM F3411-22a NetRID DSS interoperability
DSS0130,2,bASTM F3411-22a NetRID DSS interoperability
DSS0130,2,cASTM F3411-22a NetRID DSS interoperability
DSS0130,2,dASTM F3411-22a NetRID DSS interoperability
DSS0130,2,fASTM F3411-22a NetRID DSS interoperability
DSS0130,3,aASTM F3411-22a NetRID DSS interoperability
DSS0130,3,bASTM F3411-22a NetRID DSS interoperability
DSS0130,3,cASTM F3411-22a NetRID DSS interoperability
DSS0130,3,dASTM F3411-22a NetRID DSS interoperability
DSS0210ASTM F3411-22a NetRID DSS interoperability
NET0260ASTM NetRID nominal behavior
NET0260,Table1,1ASTM NetRID nominal behavior
NET0260,Table1,1aASTM NetRID nominal behavior
NET0260,Table1,23ASTM NetRID nominal behavior
NET0260,Table1,24ASTM NetRID nominal behavior
NET0260,Table1,25ASTM NetRID nominal behavior
NET0260,Table1,26ASTM NetRID nominal behavior
NET0260,Table1,7ASTM NetRID nominal behavior
NET0260,Table1,9ASTM NetRID nominal behavior
NET0270ASTM NetRID nominal behavior
NET0290ASTM NetRID nominal behavior
NET0430ASTM NetRID nominal behavior
NET0470ASTM NetRID nominal behavior
NET0480ASTM NetRID nominal behavior
NET0490ASTM NetRID nominal behavior
NET0500ASTM NetRID nominal behavior
NET0610ASTM NetRID nominal behavior
NET0710ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior
astm.f3548.v21DSS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0310Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
GEN0500Validation of operational intents
OPIN0015Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0020Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0025Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
OPIN0030Validation of operational intents
OPIN0040Validation of operational intents
SCD0015Nominal planning: conflict with higher priority
SCD0020Nominal planning: conflict with higher priority
SCD0025Nominal planning: conflict with higher priority
SCD0030Nominal planning: conflict with higher priority
SCD0035Nominal planning: not permitted conflict with equal priority
SCD0040Nominal planning: not permitted conflict with equal priority
SCD0045Nominal planning: not permitted conflict with equal priority
SCD0050Nominal planning: not permitted conflict with equal priority
USS0005Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
USS0105Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
interuss.automated_testing.flight_planningClearAreaValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
DeleteFlightSuccessValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation
ExpectedBehaviorValidation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation
interuss.automated_testing.rid.injectionDeleteTestSuccessASTM NetRID nominal behavior
ExpectedBehaviorASTM NetRID nominal behavior
UpsertTestResultASTM NetRID nominal behavior
UpsertTestSuccessASTM NetRID nominal behavior
interuss.automated_testing.rid.observationObservationSuccessASTM NetRID nominal behavior
UniqueFlightsASTM NetRID nominal behavior
From 8df8d92be4064a8799cc20dc7510221234a36bbd Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 17:37:09 +0000 Subject: [PATCH 6/8] Add requirement implementation status --- monitoring/uss_qualifier/scenarios/README.md | 2 + .../astm/netrid/v19/aggregate_checks.md | 16 +++ .../astm/netrid/v22a/aggregate_checks.md | 17 ++- .../scenarios/documentation/definitions.py | 1 + .../scenarios/documentation/parsing.py | 9 +- .../suites/astm/netrid/f3411_19.md | 70 ++++++++++- .../suites/astm/netrid/f3411_19_suite1.md | 4 + .../suites/astm/netrid/f3411_22a.md | 77 +++++++++++- .../suites/astm/netrid/f3411_22a_suite1.md | 5 + .../uss_qualifier/suites/astm/utm/f3548_21.md | 42 +++++-- .../suites/documentation/documentation.py | 61 +++++++-- .../suites/faa/uft/message_signing.md | 42 +++++-- .../suites/uspace/flight_auth.md | 42 +++++-- .../suites/uspace/network_identification.md | 76 +++++++++++- .../suites/uspace/required_services.md | 117 ++++++++++++++++-- 15 files changed, 528 insertions(+), 53 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/README.md b/monitoring/uss_qualifier/scenarios/README.md index 75a5986a26..09b920b3eb 100644 --- a/monitoring/uss_qualifier/scenarios/README.md +++ b/monitoring/uss_qualifier/scenarios/README.md @@ -87,6 +87,8 @@ A check should document the requirement(s) violated if the check fails. Require Any requirements identified (e.g., `**astm.f3411.v19.NET0420**`) must be documented as well. See [the requirements documentation](../requirements/README.md) for more information. +If the text of this section includes `TODO:`, then the check will be indicated as in development rather than complete. Documentation of intended checks with, e.g., `TODO: Implement` prior to the start of Python development is highly encouraged. + ### Cleanup phase If a test scenario wants to perform a cleanup procedure follow any non-error termination of the rest of the scenario, it must: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 8dcbdfead6..43b9e26eca 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -42,3 +42,19 @@ of the durations for the subsequent display data queries do not exceed the respe **[astm.f3411.v19.NET0260-a](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles of the durations for the replies to requested flights in an area do not exceed the respective thresholds `NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). + +## Mock USS interactions evaluation test case + +In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify +compliance with requirements. + +### Get mock USS interactions test step + +### Evaluate mock USS interactions test step + +#### No large Display Provider queries check + +If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed +area requested, then that participant will have violated **[astm.f3411.v19.NET0240](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement this check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 17760ea7eb..6338a656fd 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -4,7 +4,6 @@ In this special scenario, the report of previously executed ASTM F3411-22a NetRID scenario(s) are evaluated for the performances of the queries made during their execution. - ## Resources ### report_resource @@ -43,3 +42,19 @@ of the durations for the subsequent display data queries do not exceed the respe **[astm.f3411.v22a.NET0260-a](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles of the durations for the replies to requested flights in an area do not exceed the respective thresholds `NetSpDataResponseTime95thPercentile` (1 second) and `NetSpDataResponseTime99thPercentile` (3 seconds). + +## Mock USS interactions evaluation test case + +In this test case, the interactions with a mock_uss instance (if provided) are obtained and then examined to verify +compliance with requirements. + +### Get mock USS interactions test step + +### Evaluate mock USS interactions test step + +#### No large Display Provider queries check + +If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed +area requested, then that participant will have violated **[astm.f3411.v22a.NET0240](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement this check diff --git a/monitoring/uss_qualifier/scenarios/documentation/definitions.py b/monitoring/uss_qualifier/scenarios/documentation/definitions.py index 1cde8ac14d..ef69179a87 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/definitions.py +++ b/monitoring/uss_qualifier/scenarios/documentation/definitions.py @@ -8,6 +8,7 @@ class TestCheckDocumentation(ImplicitDict): name: str url: Optional[str] = None applicable_requirements: List[RequirementID] + has_todo: bool class TestStepDocumentation(ImplicitDict): diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index 377be9c5d0..a90227a53c 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -40,6 +40,8 @@ def _text_of(value) -> str: result += _text_of(child) return result elif isinstance(value, marko.inline.InlineElement): + if isinstance(value, marko.inline.LineBreak): + return "\n" if isinstance(value.children, str): return value.children result = "" @@ -75,17 +77,22 @@ def _parse_test_check( ) -> TestCheckDocumentation: name = _text_of(values[0])[0 : -len(TEST_CHECK_SUFFIX)] url = _url_of(doc_filename + anchors[values[0]]) + has_todo = False reqs: List[str] = [] c = 1 while c < len(values): if isinstance(values[c], marko.block.Paragraph): + if "TODO:" in _text_of(values[c]): + has_todo = True for child in values[c].children: if isinstance(child, marko.inline.StrongEmphasis): reqs.append(RequirementID(_text_of(child))) c += 1 - return TestCheckDocumentation(name=name, url=url, applicable_requirements=reqs) + return TestCheckDocumentation( + name=name, url=url, applicable_requirements=reqs, has_todo=has_todo + ) def _get_linked_test_step( diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 067f52abb1..274f6f92e9 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -8,6 +8,7 @@ 1. Suite: [DSS instance validator](f3411_19_suite1.md) ([in-suite definition](f3411_19.yaml)) 2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) +3. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) ## Checked requirements @@ -15,194 +16,261 @@ Package Requirement + Status Checked in - astm.f3411.v19 + astm.f3411.v19 A2-6-1,1a + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,1b + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,1c + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,1d + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,2a + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,2b + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,3a + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,3b + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,3c + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,3d + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,4a + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,4b + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,5 + Implemented ASTM F3411-19 NetRID DSS interoperability A2-6-1,6 + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0030 + Implemented ASTM NetRID nominal behavior DSS0070 + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130 + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,2,a + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,2,b + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,2,c + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,2,d + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,2,f + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,3,a + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,3,b + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,3,c + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0130,3,d + Implemented ASTM F3411-19 NetRID DSS interoperability DSS0210 + Implemented ASTM F3411-19 NetRID DSS interoperability + + NET0240 + Planned + ASTM F3411-19 NetRID aggregate checks + NET0260 + Implemented ASTM NetRID nominal behavior + + NET0260-a + Implemented + ASTM F3411-19 NetRID aggregate checks + NET0270 + Implemented ASTM NetRID nominal behavior NET0290 + Implemented ASTM NetRID nominal behavior + + NET0420 + Implemented + ASTM F3411-19 NetRID aggregate checks + NET0430 + Implemented ASTM NetRID nominal behavior + + NET0440 + Implemented + ASTM F3411-19 NetRID aggregate checks + NET0470 + Implemented ASTM NetRID nominal behavior NET0480 + Implemented ASTM NetRID nominal behavior NET0490 + Implemented ASTM NetRID nominal behavior NET0500 + Implemented ASTM NetRID nominal behavior NET0610 + Implemented ASTM NetRID nominal behavior NET0710 + Implemented ASTM NetRID nominal behavior astm.f3411.v22a DSS0030 + Implemented ASTM NetRID DSS: Subscription Validation DSS0050 + Implemented ASTM NetRID DSS: Subscription Validation DSS0060 + Implemented ASTM NetRID DSS: Subscription Validation interuss.automated_testing.rid.injection DeleteTestSuccess + Implemented ASTM NetRID nominal behavior ExpectedBehavior + Implemented ASTM NetRID nominal behavior UpsertTestResult + Implemented ASTM NetRID nominal behavior UpsertTestSuccess + Implemented ASTM NetRID nominal behavior interuss.automated_testing.rid.observation ObservationSuccess + Implemented ASTM NetRID nominal behavior UniqueFlights + Implemented ASTM NetRID nominal behavior diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md index f5ce9031c2..2fe606005f 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19_suite1.md @@ -12,19 +12,23 @@ Defined in [parent suite](f3411_19.md) [`suites.astm.netrid.f3411_19`](./f3411_1 Package Requirement + Status Checked in astm.f3411.v22a DSS0030 + Implemented ASTM NetRID DSS: Subscription Validation DSS0050 + Implemented ASTM NetRID DSS: Subscription Validation DSS0060 + Implemented ASTM NetRID DSS: Subscription Validation diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 5a8e2e8d20..e466936d21 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -8,6 +8,7 @@ 1. Suite: [DSS instance validator](f3411_22a_suite1.md) ([in-suite definition](f3411_22a.yaml)) 2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) 3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) +3. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) ## Checked requirements @@ -15,221 +16,295 @@ Package Requirement + Status Checked in - astm.f3411.v22a + astm.f3411.v22a A2-6-1,1a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1c + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1d + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,2a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,2b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3c + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3d + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,4a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,4b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,5 + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,6 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0030 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 + Implemented ASTM NetRID DSS: Subscription Validation DSS0060 + Implemented ASTM NetRID DSS: Subscription Validation DSS0070 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,a + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,b + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,c + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,d + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,f + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,a + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,b + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,c + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,d + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0210 + Implemented ASTM F3411-22a NetRID DSS interoperability + + NET0240 + Planned + ASTM F3411-22a NetRID aggregate checks + NET0260 + Implemented ASTM NetRID nominal behavior NET0260,Table1,1 + Implemented ASTM NetRID nominal behavior NET0260,Table1,1a + Implemented ASTM NetRID nominal behavior NET0260,Table1,23 + Implemented ASTM NetRID nominal behavior NET0260,Table1,24 + Implemented ASTM NetRID nominal behavior NET0260,Table1,25 + Implemented ASTM NetRID nominal behavior NET0260,Table1,26 + Implemented ASTM NetRID nominal behavior NET0260,Table1,7 + Implemented ASTM NetRID nominal behavior NET0260,Table1,9 + Implemented ASTM NetRID nominal behavior + + NET0260-a + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0270 + Implemented ASTM NetRID nominal behavior NET0290 + Implemented ASTM NetRID nominal behavior + + NET0420 + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0430 + Implemented ASTM NetRID nominal behavior + + NET0440 + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0470 + Implemented ASTM NetRID nominal behavior NET0480 + Implemented ASTM NetRID nominal behavior NET0490 + Implemented ASTM NetRID nominal behavior NET0500 + Implemented ASTM NetRID nominal behavior NET0610 + Implemented ASTM NetRID nominal behavior NET0710 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior interuss.automated_testing.rid.injection DeleteTestSuccess + Implemented ASTM NetRID nominal behavior ExpectedBehavior + Implemented ASTM NetRID nominal behavior UpsertTestResult + Implemented ASTM NetRID nominal behavior UpsertTestSuccess + Implemented ASTM NetRID nominal behavior interuss.automated_testing.rid.observation ObservationSuccess + Implemented ASTM NetRID nominal behavior UniqueFlights + Implemented ASTM NetRID nominal behavior diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md index 222e6145e6..0c90a2bfb6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a_suite1.md @@ -13,23 +13,28 @@ Defined in [parent suite](f3411_22a.md) [`suites.astm.netrid.f3411_22a`](./f3411 Package Requirement + Status Checked in astm.f3411.v22a DSS0030 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation DSS0050 + Implemented ASTM NetRID DSS: Subscription Validation DSS0060 + Implemented ASTM NetRID DSS: Subscription Validation NET0710 + Implemented ASTM NetRID DSS: Simple ISA diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 8b595b012e..d27cf50d52 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -17,92 +17,114 @@ Package Requirement + Status Checked in astm.f3548.v21 DSS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0310 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0500 + Implemented Validation of operational intents OPIN0015 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0020 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0025 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0030 + Implemented Validation of operational intents OPIN0040 + Implemented Validation of operational intents SCD0015 + Implemented Nominal planning: conflict with higher priority SCD0020 + Implemented Nominal planning: conflict with higher priority SCD0025 + Implemented Nominal planning: conflict with higher priority SCD0030 + Implemented Nominal planning: conflict with higher priority SCD0035 + Implemented Nominal planning: not permitted conflict with equal priority SCD0040 + Implemented Nominal planning: not permitted conflict with equal priority SCD0045 + Implemented Nominal planning: not permitted conflict with equal priority SCD0050 + Implemented Nominal planning: not permitted conflict with equal priority USS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents USS0105 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents interuss.automated_testing.flight_planning ClearArea - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents DeleteFlightSuccess - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents ExpectedBehavior - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index e0bdbdef7f..c708113606 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -96,6 +96,7 @@ def make_test_suite_documentation( lines.append("## Actions") lines.append("") base_path = os.path.dirname(suite_yaml_file) + i = 0 for i, action in enumerate(suite_def.actions): lines.extend( _render_action( @@ -110,6 +111,23 @@ def make_test_suite_documentation( ), ) ) + if ( + "report_evaluation_scenario" in suite_def + and suite_def.report_evaluation_scenario + ): + lines.extend( + _render_scenario( + suite_def.report_evaluation_scenario.scenario_type, + TestSuiteRenderContext( + parent_yaml_file=suite_yaml_file, + parent_doc_file=suite_doc_file, + base_path=base_path, + list_index=i + 1, + indent=0, + test_suites=test_suites, + ), + ) + ) lines.append("") lines.append("## Checked requirements") @@ -125,6 +143,7 @@ def make_test_suite_documentation( lines.append(" ") lines.append(" Package") lines.append(" Requirement") + lines.append(" Status") lines.append(" Checked in") lines.append(" ") @@ -143,13 +162,28 @@ def make_test_suite_documentation( for req_id in sorted(req_ids_by_package[package]): req_text = f'{req_id.requirement_name()}' + has_todo = False + has_complete = False + scenarios = {} + for checked_in in reqs[req_id].checked_in: + if checked_in.scenario.local_path not in scenarios: + scenarios[checked_in.scenario.name] = checked_in.scenario + if checked_in.check.has_todo: + has_todo = True + else: + has_complete = True + if has_complete and not has_todo: + status_text = "Implemented" + elif has_todo and not has_complete: + status_text = "Planned" + elif has_todo and has_complete: + status_text = "In progress" + else: + status_text = "Not implemented" checked_in = list( - set( - f'{c.scenario.name}' - for c in reqs[req_id].checked_in - ) + f'{scenarios[s].name}' + for s in sorted(scenarios) ) - checked_in.sort() checked_in_text = f"{'
'.join(checked_in)}" lines.append(" ") @@ -157,6 +191,7 @@ def make_test_suite_documentation( lines.append(package_line) package_line = None lines.append(f" {req_text}") + lines.append(f" {status_text}") lines.append(f" {checked_in_text}") lines.append(" ") lines.append("") @@ -297,13 +332,25 @@ def _collect_requirements_from_suite_def( suite_def: TestSuiteDefinition, ) -> Dict[RequirementID, RequirementInSuite]: reqs: Dict[RequirementID, RequirementInSuite] = {} - for action in suite_def.actions: - new_reqs = _collect_requirements_from_action(action) + + def combine(new_reqs: Dict[RequirementID, RequirementInSuite]) -> None: for req_id, req in new_reqs.items(): if req_id not in reqs: reqs[req_id] = req else: reqs[req_id].extend(req) + + for action in suite_def.actions: + combine(_collect_requirements_from_action(action)) + if ( + "report_evaluation_scenario" in suite_def + and suite_def.report_evaluation_scenario + ): + combine( + _collect_requirements_from_scenario( + suite_def.report_evaluation_scenario.scenario_type + ) + ) return reqs diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index c0e5999279..3968595b3f 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -14,92 +14,114 @@ Package Requirement + Status Checked in astm.f3548.v21 DSS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0310 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0500 + Implemented Validation of operational intents OPIN0015 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0020 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0025 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0030 + Implemented Validation of operational intents OPIN0040 + Implemented Validation of operational intents SCD0015 + Implemented Nominal planning: conflict with higher priority SCD0020 + Implemented Nominal planning: conflict with higher priority SCD0025 + Implemented Nominal planning: conflict with higher priority SCD0030 + Implemented Nominal planning: conflict with higher priority SCD0035 + Implemented Nominal planning: not permitted conflict with equal priority SCD0040 + Implemented Nominal planning: not permitted conflict with equal priority SCD0045 + Implemented Nominal planning: not permitted conflict with equal priority SCD0050 + Implemented Nominal planning: not permitted conflict with equal priority USS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents USS0105 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents interuss.automated_testing.flight_planning ClearArea - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents DeleteFlightSuccess - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents ExpectedBehavior - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 735fc86ea7..a2800dee46 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -14,92 +14,114 @@ Package Requirement + Status Checked in astm.f3548.v21 DSS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0310 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0500 + Implemented Validation of operational intents OPIN0015 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0020 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0025 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0030 + Implemented Validation of operational intents OPIN0040 + Implemented Validation of operational intents SCD0015 + Implemented Nominal planning: conflict with higher priority SCD0020 + Implemented Nominal planning: conflict with higher priority SCD0025 + Implemented Nominal planning: conflict with higher priority SCD0030 + Implemented Nominal planning: conflict with higher priority SCD0035 + Implemented Nominal planning: not permitted conflict with equal priority SCD0040 + Implemented Nominal planning: not permitted conflict with equal priority SCD0045 + Implemented Nominal planning: not permitted conflict with equal priority SCD0050 + Implemented Nominal planning: not permitted conflict with equal priority USS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents USS0105 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents interuss.automated_testing.flight_planning ClearArea - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents DeleteFlightSuccess - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation + Implemented + Flight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents ExpectedBehavior - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation + Implemented + Flight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index 26c7405cf8..7d84eea167 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -12,221 +12,295 @@ Package Requirement + Status Checked in - astm.f3411.v22a + astm.f3411.v22a A2-6-1,1a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1c + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1d + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,2a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,2b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3c + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3d + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,4a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,4b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,5 + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,6 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0030 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 + Implemented ASTM NetRID DSS: Subscription Validation DSS0060 + Implemented ASTM NetRID DSS: Subscription Validation DSS0070 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,a + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,b + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,c + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,d + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,f + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,a + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,b + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,c + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,d + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0210 + Implemented ASTM F3411-22a NetRID DSS interoperability + + NET0240 + Planned + ASTM F3411-22a NetRID aggregate checks + NET0260 + Implemented ASTM NetRID nominal behavior NET0260,Table1,1 + Implemented ASTM NetRID nominal behavior NET0260,Table1,1a + Implemented ASTM NetRID nominal behavior NET0260,Table1,23 + Implemented ASTM NetRID nominal behavior NET0260,Table1,24 + Implemented ASTM NetRID nominal behavior NET0260,Table1,25 + Implemented ASTM NetRID nominal behavior NET0260,Table1,26 + Implemented ASTM NetRID nominal behavior NET0260,Table1,7 + Implemented ASTM NetRID nominal behavior NET0260,Table1,9 + Implemented ASTM NetRID nominal behavior + + NET0260-a + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0270 + Implemented ASTM NetRID nominal behavior NET0290 + Implemented ASTM NetRID nominal behavior + + NET0420 + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0430 + Implemented ASTM NetRID nominal behavior + + NET0440 + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0470 + Implemented ASTM NetRID nominal behavior NET0480 + Implemented ASTM NetRID nominal behavior NET0490 + Implemented ASTM NetRID nominal behavior NET0500 + Implemented ASTM NetRID nominal behavior NET0610 + Implemented ASTM NetRID nominal behavior NET0710 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior interuss.automated_testing.rid.injection DeleteTestSuccess + Implemented ASTM NetRID nominal behavior ExpectedBehavior + Implemented ASTM NetRID nominal behavior UpsertTestResult + Implemented ASTM NetRID nominal behavior UpsertTestSuccess + Implemented ASTM NetRID nominal behavior interuss.automated_testing.rid.observation ObservationSuccess + Implemented ASTM NetRID nominal behavior UniqueFlights + Implemented ASTM NetRID nominal behavior diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index f96aa06249..ba7e196d4f 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -13,307 +13,402 @@ Package Requirement + Status Checked in - astm.f3411.v22a + astm.f3411.v22a A2-6-1,1a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1c + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,1d + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,2a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,2b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3c + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,3d + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,4a + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,4b + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,5 + Implemented ASTM F3411-22a NetRID DSS interoperability A2-6-1,6 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0030 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Subscription Validation
ASTM NetRID nominal behavior DSS0050 + Implemented ASTM NetRID DSS: Subscription Validation DSS0060 + Implemented ASTM NetRID DSS: Subscription Validation DSS0070 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130 + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,a + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,b + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,c + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,d + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,2,f + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,a + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,b + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,c + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0130,3,d + Implemented ASTM F3411-22a NetRID DSS interoperability DSS0210 + Implemented ASTM F3411-22a NetRID DSS interoperability + + NET0240 + Planned + ASTM F3411-22a NetRID aggregate checks + NET0260 + Implemented ASTM NetRID nominal behavior NET0260,Table1,1 + Implemented ASTM NetRID nominal behavior NET0260,Table1,1a + Implemented ASTM NetRID nominal behavior NET0260,Table1,23 + Implemented ASTM NetRID nominal behavior NET0260,Table1,24 + Implemented ASTM NetRID nominal behavior NET0260,Table1,25 + Implemented ASTM NetRID nominal behavior NET0260,Table1,26 + Implemented ASTM NetRID nominal behavior NET0260,Table1,7 + Implemented ASTM NetRID nominal behavior NET0260,Table1,9 + Implemented ASTM NetRID nominal behavior + + NET0260-a + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0270 + Implemented ASTM NetRID nominal behavior NET0290 + Implemented ASTM NetRID nominal behavior + + NET0420 + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0430 + Implemented ASTM NetRID nominal behavior + + NET0440 + Implemented + ASTM F3411-22a NetRID aggregate checks + NET0470 + Implemented ASTM NetRID nominal behavior NET0480 + Implemented ASTM NetRID nominal behavior NET0490 + Implemented ASTM NetRID nominal behavior NET0500 + Implemented ASTM NetRID nominal behavior NET0610 + Implemented ASTM NetRID nominal behavior NET0710 + Implemented ASTM NetRID DSS: Simple ISA
ASTM NetRID nominal behavior astm.f3548.v21 DSS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0310 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents GEN0500 + Implemented Validation of operational intents OPIN0015 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0020 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0025 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents OPIN0030 + Implemented Validation of operational intents OPIN0040 + Implemented Validation of operational intents SCD0015 + Implemented Nominal planning: conflict with higher priority SCD0020 + Implemented Nominal planning: conflict with higher priority SCD0025 + Implemented Nominal planning: conflict with higher priority SCD0030 + Implemented Nominal planning: conflict with higher priority SCD0035 + Implemented Nominal planning: not permitted conflict with equal priority SCD0040 + Implemented Nominal planning: not permitted conflict with equal priority SCD0045 + Implemented Nominal planning: not permitted conflict with equal priority SCD0050 + Implemented Nominal planning: not permitted conflict with equal priority USS0005 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents USS0105 - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents interuss.automated_testing.flight_planning ClearArea - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority + Implemented + Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents DeleteFlightSuccess - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation + Implemented + Flight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents ExpectedBehavior - Validation of operational intents
Nominal planning: not permitted conflict with equal priority
Nominal planning: conflict with higher priority
Flight authorisation validation + Implemented + Flight authorisation validation
Nominal planning: conflict with higher priority
Nominal planning: not permitted conflict with equal priority
Validation of operational intents interuss.automated_testing.rid.injection DeleteTestSuccess + Implemented ASTM NetRID nominal behavior ExpectedBehavior + Implemented ASTM NetRID nominal behavior UpsertTestResult + Implemented ASTM NetRID nominal behavior UpsertTestSuccess + Implemented ASTM NetRID nominal behavior interuss.automated_testing.rid.observation ObservationSuccess + Implemented ASTM NetRID nominal behavior UniqueFlights + Implemented ASTM NetRID nominal behavior From be369e158fa2819f4c13115cbbf9d8d1fbf753b5 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 17:55:17 +0000 Subject: [PATCH 7/8] Explicitly preload module content --- .../suites/documentation/format_documentation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/monitoring/uss_qualifier/suites/documentation/format_documentation.py b/monitoring/uss_qualifier/suites/documentation/format_documentation.py index e04d5038d6..0859588a5d 100644 --- a/monitoring/uss_qualifier/suites/documentation/format_documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/format_documentation.py @@ -3,6 +3,8 @@ import sys from implicitdict import ImplicitDict +from monitoring.monitorlib.inspection import import_submodules +from monitoring.uss_qualifier import scenarios, suites, action_generators from monitoring.uss_qualifier.fileio import load_dict_with_references from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition from monitoring.uss_qualifier.suites.documentation.documentation import ( @@ -12,6 +14,10 @@ def main(lint: bool) -> int: + import_submodules(scenarios) + import_submodules(suites) + import_submodules(action_generators) + test_suite_docs = {} for suite_yaml_file in find_test_suites(): suite_def: TestSuiteDefinition = ImplicitDict.parse( From c106391038bbd365e900d25cf93f904a71121f09 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 11 Sep 2023 18:25:03 +0000 Subject: [PATCH 8/8] Fix tests --- .../flight_authorization/general_flight_authorization.py | 1 + .../interuss/geospatial_map/geospatial_feature_comprehension.py | 1 + monitoring/uss_qualifier/scenarios/scenario.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py index 6024aa62b1..81f5fb4928 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py @@ -92,6 +92,7 @@ def _plan_flights(self): name=c.name, url=c.url, applicable_requirements=row.requirement_ids, + has_todo=False, ) for c in checks ] diff --git a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py index efb0adc6ee..1ba899cd98 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py +++ b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py @@ -63,6 +63,7 @@ def _map_query(self): name=check_name, url=check_url, applicable_requirements=row.requirement_ids, + has_todo=False, ) doc = TestStepDocumentation( name=row.geospatial_check_id, diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index c564045ba0..78a0791fb3 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -364,7 +364,7 @@ def check( check_list = ", ".join(available_checks) if self._allow_undocumented_checks: check_documentation = TestCheckDocumentation( - name=name, applicable_requirements=[] + name=name, applicable_requirements=[], has_todo=False ) else: raise RuntimeError(