[uss_qualifier] Improve action generator & test suite documentation (#…

…194)

* Generate test suite documentation for action generators

* Generate documentation for in-suite suite definitions

* Remove orphaned documentation

* Replace long arg lists with context

* Add checked requirements

* Add requirement implementation status

* Explicitly preload module content

* Fix tests

monitoring/uss_qualifier/action_generators/action_generator.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from abc import ABC, abstractmethod
 import inspect
-from typing import Generic, Dict, Optional, TypeVar
+from typing import Generic, Dict, Optional, TypeVar, List, Type
 
 from implicitdict import ImplicitDict
 from monitoring import uss_qualifier as uss_qualifier_module
@@ -12,6 +12,10 @@
 from monitoring.uss_qualifier.action_generators.definitions import (
     ActionGeneratorSpecificationType,
     ActionGeneratorDefinition,
+    GeneratorTypeName,
+)
+from monitoring.uss_qualifier.action_generators.documentation.definitions import (
+    PotentialGeneratedAction,
 )
 from monitoring.uss_qualifier.reports.report import TestSuiteActionReport
 from monitoring.uss_qualifier.resources.definitions import ResourceID
@@ -45,32 +49,35 @@ def run_next_action(self) -> Optional[TestSuiteActionReport]:
             "A concrete action generator must implement `actions` method"
         )
 
+    @classmethod
+    def list_potential_actions(
+        cls, specification: Optional[ActionGeneratorSpecificationType]
+    ) -> List[PotentialGeneratedAction]:
+        """Enumerate the potential actions that may be run by an instance of this generator type.
+
+        Concrete subclasses of ActionGenerator must implement a classmethod that shadows this one according to this
+        specification.
+
+        Args:
+            specification: A serializable (subclass of implicitdict.ImplicitDict) specification for how to create the
+                action generator instance, or None if the action generator type does not need a specification.
+
+        Returns: All potential actions that may be generated by this generator, depending on the resources provided.
+        """
+        raise NotImplementedError(
+            "A concrete action generator must implement `list_potential_actions` classmethod"
+        )
+
     @staticmethod
     def make_from_definition(
         definition: ActionGeneratorDefinition, resources: Dict[ResourceID, ResourceType]
     ) -> ActionGeneratorType:
-        from monitoring.uss_qualifier import (
-            action_generators as action_generators_module,
+        action_generator_type = action_generator_type_from_name(
+            definition.generator_type
         )
+        specification_type = action_generator_specification_type(action_generator_type)
 
-        import_submodules(action_generators_module)
-        action_generator_type = get_module_object_by_name(
-            parent_module=uss_qualifier_module,
-            object_name=definition.generator_type,
-        )
-        if not issubclass(action_generator_type, ActionGenerator):
-            raise NotImplementedError(
-                "Action generator type {} is not a subclass of the ActionGenerator base class".format(
-                    action_generator_type.__name__
-                )
-            )
-        constructor_signature = inspect.signature(action_generator_type.__init__)
-        specification_type = None
         constructor_args = {}
-        for arg_name, arg in constructor_signature.parameters.items():
-            if arg_name == "specification":
-                specification_type = arg.annotation
-                break
         if specification_type is not None:
             constructor_args["specification"] = ImplicitDict.parse(
                 definition.specification, specification_type
@@ -82,3 +89,36 @@ def make_from_definition(
 
 
 ActionGeneratorType = TypeVar("ActionGeneratorType", bound=ActionGenerator)
+
+
+def action_generator_type_from_name(
+    action_generator_type_name: GeneratorTypeName,
+) -> Type[ActionGenerator]:
+    from monitoring.uss_qualifier import (
+        action_generators as action_generators_module,
+    )
+
+    import_submodules(action_generators_module)
+    action_generator_type = get_module_object_by_name(
+        parent_module=uss_qualifier_module,
+        object_name=action_generator_type_name,
+    )
+    if not issubclass(action_generator_type, ActionGenerator):
+        raise NotImplementedError(
+            "Action generator type {} is not a subclass of the ActionGenerator base class".format(
+                action_generator_type.__name__
+            )
+        )
+    return action_generator_type
+
+
+def action_generator_specification_type(
+    action_generator_type: Type[ActionGenerator],
+) -> Optional[Type]:
+    constructor_signature = inspect.signature(action_generator_type.__init__)
+    specification_type = None
+    for arg_name, arg in constructor_signature.parameters.items():
+        if arg_name == "specification":
+            specification_type = arg.annotation
+            break
+    return specification_type

monitoring/uss_qualifier/action_generators/astm/f3411/for_each_dss.py

@@ -3,6 +3,12 @@
 from implicitdict import ImplicitDict
 
 from monitoring.monitorlib.inspection import fullname
+from monitoring.uss_qualifier.action_generators.documentation.definitions import (
+    PotentialGeneratedAction,
+)
+from monitoring.uss_qualifier.action_generators.documentation.documentation import (
+    list_potential_actions_for_action_declaration,
+)
 from monitoring.uss_qualifier.reports.report import TestSuiteActionReport
 from monitoring.uss_qualifier.resources.astm.f3411 import (
     DSSInstanceResource,
@@ -34,6 +40,14 @@ class ForEachDSS(ActionGenerator[ForEachDSSSpecification]):
     _current_action: int
     _failure_reaction: ReactionToFailure
 
+    @classmethod
+    def list_potential_actions(
+        cls, specification: ForEachDSSSpecification
+    ) -> List[PotentialGeneratedAction]:
+        return list_potential_actions_for_action_declaration(
+            specification.action_to_repeat
+        )
+
     def __init__(
         self,
         specification: ForEachDSSSpecification,

monitoring/uss_qualifier/action_generators/definitions.py

@@ -5,7 +5,7 @@
 
 
 GeneratorTypeName = str
-"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier.action_generators` module"""
+"""This plain string represents a type of action generator, expressed as a Python class name qualified relative to the `uss_qualifier` module"""
 
 
 ActionGeneratorSpecificationType = TypeVar(

monitoring/uss_qualifier/action_generators/documentation/definitions.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+from implicitdict import ImplicitDict
+from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName
+from monitoring.uss_qualifier.fileio import FileReference
+from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName
+from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition
+
+
+class PotentialTestScenarioAction(ImplicitDict):
+    scenario_type: TestScenarioTypeName
+    """Type of test scenario."""
+
+
+class PotentialTestSuiteAction(ImplicitDict):
+    suite_type: Optional[FileReference]
+    """Type/location of test suite.  Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder"""
+
+    suite_definition: Optional[TestSuiteDefinition]
+    """Definition of test suite internal to the configuration -- specified instead of `suite_type`."""
+
+
+class PotentialActionGeneratorAction(ImplicitDict):
+    generator_type: GeneratorTypeName
+    """Type of action generator."""
+
+    specification: dict
+    """Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`"""
+
+
+class PotentialGeneratedAction(ImplicitDict):
+    test_scenario: Optional[PotentialTestScenarioAction]
+    test_suite: Optional[PotentialTestSuiteAction]
+    action_generator: Optional[PotentialActionGeneratorAction]
+
+    def get_action_type(self) -> ActionType:
+        matches = [v for v in ActionType if v in self and self[v]]
+        if len(matches) != 1:
+            ActionType.raise_invalid_action_declaration()
+        return ActionType(matches[0])

monitoring/uss_qualifier/action_generators/documentation/documentation.py

@@ -0,0 +1,78 @@
+from typing import List, Union
+
+from implicitdict import ImplicitDict
+from monitoring.uss_qualifier.action_generators.action_generator import (
+    action_generator_type_from_name,
+    action_generator_specification_type,
+)
+from monitoring.uss_qualifier.action_generators.definitions import (
+    ActionGeneratorDefinition,
+)
+from monitoring.uss_qualifier.action_generators.documentation.definitions import (
+    PotentialGeneratedAction,
+    PotentialTestScenarioAction,
+    PotentialTestSuiteAction,
+    PotentialActionGeneratorAction,
+)
+from monitoring.uss_qualifier.suites.definitions import (
+    TestSuiteActionDeclaration,
+    ActionType,
+)
+
+
+def list_potential_actions_for_action_generator_definition(
+    generator_def: Union[ActionGeneratorDefinition, PotentialActionGeneratorAction]
+) -> List[PotentialGeneratedAction]:
+    action_generator_type = action_generator_type_from_name(
+        generator_def.generator_type
+    )
+    specification_type = action_generator_specification_type(action_generator_type)
+    if specification_type is not None:
+        spec = ImplicitDict.parse(generator_def.specification, specification_type)
+    else:
+        spec = None
+    return action_generator_type.list_potential_actions(spec)
+
+
+def list_potential_actions_for_action_declaration(
+    declaration: TestSuiteActionDeclaration,
+) -> List[PotentialGeneratedAction]:
+    action_type = declaration.get_action_type()
+    if action_type == ActionType.TestScenario:
+        return [
+            PotentialGeneratedAction(
+                test_scenario=PotentialTestScenarioAction(
+                    scenario_type=declaration.test_scenario.scenario_type
+                )
+            )
+        ]
+    elif action_type == ActionType.TestSuite:
+        if "suite_type" in declaration.test_suite and declaration.test_suite.suite_type:
+            return [
+                PotentialGeneratedAction(
+                    test_suite=PotentialTestSuiteAction(
+                        suite_type=declaration.test_suite.suite_type
+                    )
+                )
+            ]
+        elif (
+            "suite_definition" in declaration.test_suite
+            and declaration.test_suite.suite_definition
+        ):
+            return [
+                PotentialGeneratedAction(
+                    test_suite=PotentialTestSuiteAction(
+                        suite_definition=declaration.test_suite.suite_definition
+                    )
+                )
+            ]
+    elif action_type == ActionType.ActionGenerator:
+        return [
+            PotentialGeneratedAction(
+                action_generator=PotentialActionGeneratorAction(
+                    generator_type=declaration.action_generator.generator_type
+                )
+            )
+        ]
+    else:
+        raise NotImplementedError(f"Action type {action_type} is not supported")

monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py

@@ -3,6 +3,17 @@
 from implicitdict import ImplicitDict
 
 from monitoring.monitorlib.inspection import fullname
+from monitoring.uss_qualifier.action_generators.action_generator import (
+    action_generator_type_from_name,
+    action_generator_specification_type,
+)
+from monitoring.uss_qualifier.action_generators.documentation.definitions import (
+    PotentialGeneratedAction,
+    PotentialTestScenarioAction,
+)
+from monitoring.uss_qualifier.action_generators.documentation.documentation import (
+    list_potential_actions_for_action_declaration,
+)
 from monitoring.uss_qualifier.reports.report import TestSuiteActionReport
 from monitoring.uss_qualifier.resources.definitions import ResourceID
 from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource
@@ -14,7 +25,10 @@
     ResourceType,
 )
 
-from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration
+from monitoring.uss_qualifier.suites.definitions import (
+    TestSuiteActionDeclaration,
+    ActionType,
+)
 from monitoring.uss_qualifier.suites.suite import (
     ActionGenerator,
     TestSuiteAction,
@@ -43,6 +57,14 @@ class FlightPlannerCombinations(
     _current_action: int
     _failure_reaction: ReactionToFailure
 
+    @classmethod
+    def list_potential_actions(
+        cls, specification: FlightPlannerCombinationsSpecification
+    ) -> List[PotentialGeneratedAction]:
+        return list_potential_actions_for_action_declaration(
+            specification.action_to_repeat
+        )
+
     def __init__(
         self,
         specification: FlightPlannerCombinationsSpecification,

monitoring/uss_qualifier/action_generators/repetition/repeat.py

@@ -1,6 +1,12 @@
 from typing import Dict, List, Optional
 
 from implicitdict import ImplicitDict
+from monitoring.uss_qualifier.action_generators.documentation.definitions import (
+    PotentialGeneratedAction,
+)
+from monitoring.uss_qualifier.action_generators.documentation.documentation import (
+    list_potential_actions_for_action_declaration,
+)
 from monitoring.uss_qualifier.reports.report import TestSuiteActionReport
 from monitoring.uss_qualifier.resources.definitions import ResourceID
 from monitoring.uss_qualifier.resources.resource import ResourceType
@@ -26,6 +32,14 @@ class Repeat(ActionGenerator[RepeatSpecification]):
     _current_action: int
     _failure_reaction: ReactionToFailure
 
+    @classmethod
+    def list_potential_actions(
+        cls, specification: RepeatSpecification
+    ) -> List[PotentialGeneratedAction]:
+        return list_potential_actions_for_action_declaration(
+            specification.action_to_repeat
+        )
+
     def __init__(
         self,
         specification: RepeatSpecification,

monitoring/uss_qualifier/reports/report.py

@@ -7,6 +7,7 @@
 from implicitdict import ImplicitDict, StringBasedDateTime
 
 from monitoring.monitorlib import fetch, inspection
+from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName
 from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.configurations.configuration import (
     TestConfiguration,
@@ -18,6 +19,7 @@
     JSONPathExpression,
 )
 from monitoring.uss_qualifier.requirements.definitions import RequirementID
+from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName
 
 
 class FailedCheck(ImplicitDict):
@@ -211,7 +213,7 @@ class TestScenarioReport(ImplicitDict):
     name: str
     """Name of this test scenario"""
 
-    scenario_type: str
+    scenario_type: TestScenarioTypeName
     """Type of this test scenario"""
 
     documentation_url: str
@@ -295,7 +297,7 @@ def participant_ids(self) -> Set[ParticipantID]:
 
 
 class ActionGeneratorReport(ImplicitDict):
-    generator_type: str
+    generator_type: GeneratorTypeName
     """Type of action generator"""
 
     actions: List[TestSuiteActionReport]

monitoring/uss_qualifier/scenarios/README.md

@@ -87,6 +87,8 @@ A check should document the requirement(s) violated if the check fails.  Require
 
 Any requirements identified (e.g., `**astm.f3411.v19.NET0420**`) must be documented as well.  See [the requirements documentation](../requirements/README.md) for more information.
 
+If the text of this section includes `TODO:`, then the check will be indicated as in development rather than complete.  Documentation of intended checks with, e.g., `TODO: Implement` prior to the start of Python development is highly encouraged.
+
 ### Cleanup phase
 
 If a test scenario wants to perform a cleanup procedure follow any non-error termination of the rest of the scenario, it must: