[uss_qualifier] Add execution control (#292)

* Add execution control

* Fix behavior with missing execution specification

monitoring/uss_qualifier/action_generators/documentation/definitions.py

@@ -4,7 +4,11 @@
 from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName
 from monitoring.uss_qualifier.fileio import FileReference
 from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName
-from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition
+from monitoring.uss_qualifier.suites.definitions import (
+    ActionType,
+    TestSuiteDefinition,
+    TestSuiteTypeName,
+)
 
 
 class PotentialTestScenarioAction(ImplicitDict):
@@ -13,7 +17,7 @@ class PotentialTestScenarioAction(ImplicitDict):
 
 
 class PotentialTestSuiteAction(ImplicitDict):
-    suite_type: Optional[FileReference]
+    suite_type: Optional[TestSuiteTypeName]
     """Type/location of test suite.  Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder"""
 
     suite_definition: Optional[TestSuiteDefinition]

monitoring/uss_qualifier/configurations/README.md

@@ -2,7 +2,7 @@
 
 ## Usage
 
-To execute a test run with uss_qualifier, a uss_qualifier configuration must be provided.  This configuration consists of the test suite to run, along with definitions for all resources needed by that test suite, plus information about artifacts that should be generated.  See [`USSQualifierConfiguration`](configuration.py) for the exact schema.
+To execute a test run with uss_qualifier, a uss_qualifier configuration must be provided.  This configuration consists of the test suite to run, along with definitions for all resources needed by that test suite, plus information about artifacts that should be generated.  See [`USSQualifierConfiguration`](configuration.py) for the exact schema and [the dev configurations](./dev) for examples.
 
 ### Specifying
 
@@ -67,10 +67,141 @@ Loading _q.json_ results in the object:
 
 More details may be found in [`fileio.py`](../fileio.py).
 
+## Execution control
+
+To skip or selectively execute portions of a test run defined by a configuration, populate [the `execution` field of the `TestConfiguration`](configuration.py).  This field controls execution of portions of the test run by skipping actions according to specified criteria.  When debugging, this feature can be used to selectively execute only a scenario (or set of scenarios) of interest, or exclude a problematic scenario (or set of scenarios) from execution.  Some examples are shown below:
+
+### Skip all test scenarios:
+
+_Shows test suite / action generator structure_
+
+```yaml
+execution:
+  skip_action_when:
+    - is_test_scenario: {}
+```
+
+### Skip a particular test suite
+
+```yaml
+execution:
+  skip_action_when:
+    - is_test_suite:
+        types: [suites.astm.netrid.f3411_22a]
+```
+
+### Only run two kinds of scenarios
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - is_test_scenario:
+        types: [scenarios.interuss.mock_uss.configure_locality.ConfigureLocality, scenarios.astm.utm.FlightIntentValidation]
+```
+
+### Only run the first, ninth, and tenth test scenarios in the test run
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - nth_instance:
+        n: [{i: 1}, {lo: 9, hi: 10}]
+        where_action:
+            is_test_scenario: {}
+```
+
+### Only run test scenarios with a matching name
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - is_test_scenario: {}
+      regex_matches_name: 'ASTM NetRID DSS: Simple ISA'
+```
+
+### Run everything except two kinds of test suites
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+      except_when:
+        - regex_matches_name: 'ASTM F3411-22a'
+        - is_test_suite:
+            types: [suites.astm.utm.f3548_21]
+    - is_test_scenario: {}
+```
+
+### Only run the immediate test scenario children of a particular test suite
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - is_test_scenario:
+      has_ancestor:
+        of_generation: 1
+        which:
+          - is_test_suite: {}
+            regex_matches_name: 'DSS testing for ASTM NetRID F3548-21'
+```
+
+### Only run test scenarios that are descendants of a particular test suite
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - is_test_scenario:
+      has_ancestor:
+        which:
+          - is_test_suite:
+              types: [suites.astm.utm.f3548_21]
+```
+
+### Only run the third instance of a particular test scenario name
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - nth_instance:
+        n: [{i: 3}]
+        where_action:
+          regex_matches_name: 'Nominal planning: conflict with higher priority'
+```
+
+### Only run the test scenarios for the second instance of a particular named action generator
+
+```yaml
+execution:
+  include_action_when:
+    - is_action_generator: {}
+    - is_test_suite: {}
+    - is_test_scenario: {}
+      has_ancestor:
+        which:
+          - nth_instance:
+              n: [{i: 2}]
+              where_action:
+                is_action_generator: {}
+                regex_matches_name: 'For each appropriate combination of flight planner\(s\)'
+```
+
 ## Design notes
 
 1. Even though all the scenarios, cases, steps and checks are fully defined for a particular test suite, the scenarios require data customized for a particular ecosystem – this data is provided as "test resources" which are created from the specifications in a "test configuration".
-2. A test configuration is associated with exactly one test suite, and contains descriptions for how to create each of the set of required test resources.
+2. A test configuration is associated with exactly one test action (test scenario, test suite, action generator), and contains descriptions for how to create each of the set of required test resources.
     * The resources required for a particular test definition depend on which test scenarios are included in the test suite.
 3. One resource can be used by many different test scenarios.
 4. One test scenario may use multiple resources.

monitoring/uss_qualifier/configurations/configuration.py

@@ -1,21 +1,132 @@
+from __future__ import annotations
 from typing import Optional, List, Dict
 
 from implicitdict import ImplicitDict
 
 from monitoring.monitorlib.dicts import JSONAddress
+from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName
 from monitoring.uss_qualifier.reports.validation.definitions import (
     ValidationConfiguration,
 )
 from monitoring.uss_qualifier.requirements.definitions import RequirementCollection
 from monitoring.uss_qualifier.resources.definitions import ResourceCollection
+from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName
 from monitoring.uss_qualifier.suites.definitions import (
     TestSuiteActionDeclaration,
+    TestSuiteTypeName,
 )
 
 ParticipantID = str
 """String that refers to a participant being qualified by uss_qualifier"""
 
 
+class InstanceIndexRange(ImplicitDict):
+    lo: Optional[int]
+    """If specified, no indices lower than this value will be included in the range."""
+
+    i: Optional[int]
+    """If specified, no index other than this one will be included in the range."""
+
+    hi: Optional[int]
+    """If specified, no indices higher than this value will be included in the range."""
+
+    def includes(self, i: int) -> bool:
+        if "i" in self and self.i is not None and i != self.i:
+            return False
+        if "lo" in self and self.lo is not None and i < self.lo:
+            return False
+        if "hi" in self and self.hi is not None and i > self.hi:
+            return False
+        return True
+
+
+class ActionGeneratorSelectionCondition(ImplicitDict):
+    """By default, select all action generators.  When specified, limit selection to specified conditions."""
+
+    types: Optional[List[GeneratorTypeName]]
+    """Only select action generators of the specified types."""
+
+
+class TestSuiteSelectionCondition(ImplicitDict):
+    """By default, select all test suites.  When specified, limit selection to specified conditions."""
+
+    types: Optional[List[TestSuiteTypeName]]
+    """Only select test suites of the specified types."""
+
+
+class TestScenarioSelectionCondition(ImplicitDict):
+    """By default, select all test scenarios.  When specified, limit selection to specified conditions."""
+
+    types: Optional[List[TestScenarioTypeName]]
+    """Only select test scenarios of the specified types."""
+
+
+class NthInstanceCondition(ImplicitDict):
+    """Select an action once a certain number of matching instances have happened."""
+
+    n: List[InstanceIndexRange]
+    """Only select an action if it is one of these nth instances."""
+
+    where_action: TestSuiteActionSelectionCondition
+    """Condition that an action must meet to be selected as an instance in this condition."""
+
+
+class AncestorSelectionCondition(ImplicitDict):
+    """Select ancestor actions meeting all the specified conditions."""
+
+    of_generation: Optional[int]
+    """The ancestor is exactly this many generations removed (1 = parent, 2 = grandparent, etc).
+
+    If not specified, an ancestor of any generation meeting the `which` conditions will be selected."""
+
+    which: List[TestSuiteActionSelectionCondition]
+    """Only select an ancestor meeting ALL of these conditions."""
+
+
+class TestSuiteActionSelectionCondition(ImplicitDict):
+    """Condition for selecting TestSuiteActions.
+
+    If more than one subcondition is specified, satisfaction of ALL subconditions are necessary to select the action."""
+
+    is_action_generator: Optional[ActionGeneratorSelectionCondition]
+    """Select these action generator actions."""
+
+    is_test_suite: Optional[TestSuiteSelectionCondition]
+    """Select these test suite actions."""
+
+    is_test_scenario: Optional[TestScenarioSelectionCondition]
+    """Select these test scenario actions."""
+
+    regex_matches_name: Optional[str]
+    """Select actions where this regular expression has a match in the action's name."""
+
+    defined_at: Optional[List[JSONAddress]]
+    """Select actions defined at one of the specified addresses.
+
+    The top-level action in a test run is 'test_scenario', 'test_suite', or 'action_generator'.  Children use the
+    'actions' property, but then must specify the type of the action.  So, e.g., the test scenario that is the third
+    action of a test suite which is the second action in an action generator would be
+    'action_generator.actions[1].test_suite.actions[2].test_scenario'.  An address that starts or ends with 'actions[i]'
+    is invalid and will never match."""
+
+    nth_instance: Optional[NthInstanceCondition]
+    """Select only certain instances of matching actions."""
+
+    has_ancestor: Optional[AncestorSelectionCondition]
+    """Select only actions with a matching ancestor."""
+
+    except_when: Optional[List[TestSuiteActionSelectionCondition]]
+    """Do not select actions selected by any of these conditions, even when they are selected by one or more conditions above."""
+
+
+class ExecutionConfiguration(ImplicitDict):
+    include_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None
+    """If specified, only execute test actions if they are selected by ANY of these conditions (and not selected by any of the `skip_when` conditions)."""
+
+    skip_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None
+    """If specified, do not execute test actions if they are selected by ANY of these conditions."""
+
+
 class TestConfiguration(ImplicitDict):
     action: TestSuiteActionDeclaration
     """The action this test configuration wants to run (usually a test suite)"""
@@ -26,6 +137,9 @@ class TestConfiguration(ImplicitDict):
     resources: ResourceCollection
     """Declarations for resources used by the test suite"""
 
+    execution: Optional[ExecutionConfiguration]
+    """Specification for how to execute the test run."""
+
 
 TestedRequirementsCollectionIdentifier = str
 """Identifier for a requirements collection, local to a TestedRequirementsConfiguration artifact configuration."""