diff --git a/monitoring/uss_qualifier/action_generators/documentation/definitions.py b/monitoring/uss_qualifier/action_generators/documentation/definitions.py index 33bb951e80..7557421ef4 100644 --- a/monitoring/uss_qualifier/action_generators/documentation/definitions.py +++ b/monitoring/uss_qualifier/action_generators/documentation/definitions.py @@ -4,7 +4,11 @@ from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.fileio import FileReference from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName -from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition +from monitoring.uss_qualifier.suites.definitions import ( + ActionType, + TestSuiteDefinition, + TestSuiteTypeName, +) class PotentialTestScenarioAction(ImplicitDict): @@ -13,7 +17,7 @@ class PotentialTestScenarioAction(ImplicitDict): class PotentialTestSuiteAction(ImplicitDict): - suite_type: Optional[FileReference] + suite_type: Optional[TestSuiteTypeName] """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" suite_definition: Optional[TestSuiteDefinition] diff --git a/monitoring/uss_qualifier/configurations/README.md b/monitoring/uss_qualifier/configurations/README.md index f052adf98d..a0830a3602 100644 --- a/monitoring/uss_qualifier/configurations/README.md +++ b/monitoring/uss_qualifier/configurations/README.md @@ -2,7 +2,7 @@ ## Usage -To execute a test run with uss_qualifier, a uss_qualifier configuration must be provided. This configuration consists of the test suite to run, along with definitions for all resources needed by that test suite, plus information about artifacts that should be generated. See [`USSQualifierConfiguration`](configuration.py) for the exact schema. +To execute a test run with uss_qualifier, a uss_qualifier configuration must be provided. This configuration consists of the test suite to run, along with definitions for all resources needed by that test suite, plus information about artifacts that should be generated. See [`USSQualifierConfiguration`](configuration.py) for the exact schema and [the dev configurations](./dev) for examples. ### Specifying @@ -67,10 +67,141 @@ Loading _q.json_ results in the object: More details may be found in [`fileio.py`](../fileio.py). +## Execution control + +To skip or selectively execute portions of a test run defined by a configuration, populate [the `execution` field of the `TestConfiguration`](configuration.py). This field controls execution of portions of the test run by skipping actions according to specified criteria. When debugging, this feature can be used to selectively execute only a scenario (or set of scenarios) of interest, or exclude a problematic scenario (or set of scenarios) from execution. Some examples are shown below: + +### Skip all test scenarios: + +_Shows test suite / action generator structure_ + +```yaml +execution: + skip_action_when: + - is_test_scenario: {} +``` + +### Skip a particular test suite + +```yaml +execution: + skip_action_when: + - is_test_suite: + types: [suites.astm.netrid.f3411_22a] +``` + +### Only run two kinds of scenarios + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: + types: [scenarios.interuss.mock_uss.configure_locality.ConfigureLocality, scenarios.astm.utm.FlightIntentValidation] +``` + +### Only run the first, ninth, and tenth test scenarios in the test run + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - nth_instance: + n: [{i: 1}, {lo: 9, hi: 10}] + where_action: + is_test_scenario: {} +``` + +### Only run test scenarios with a matching name + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: {} + regex_matches_name: 'ASTM NetRID DSS: Simple ISA' +``` + +### Run everything except two kinds of test suites + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + except_when: + - regex_matches_name: 'ASTM F3411-22a' + - is_test_suite: + types: [suites.astm.utm.f3548_21] + - is_test_scenario: {} +``` + +### Only run the immediate test scenario children of a particular test suite + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: + has_ancestor: + of_generation: 1 + which: + - is_test_suite: {} + regex_matches_name: 'DSS testing for ASTM NetRID F3548-21' +``` + +### Only run test scenarios that are descendants of a particular test suite + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: + has_ancestor: + which: + - is_test_suite: + types: [suites.astm.utm.f3548_21] +``` + +### Only run the third instance of a particular test scenario name + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - nth_instance: + n: [{i: 3}] + where_action: + regex_matches_name: 'Nominal planning: conflict with higher priority' +``` + +### Only run the test scenarios for the second instance of a particular named action generator + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: {} + has_ancestor: + which: + - nth_instance: + n: [{i: 2}] + where_action: + is_action_generator: {} + regex_matches_name: 'For each appropriate combination of flight planner\(s\)' +``` + ## Design notes 1. Even though all the scenarios, cases, steps and checks are fully defined for a particular test suite, the scenarios require data customized for a particular ecosystem – this data is provided as "test resources" which are created from the specifications in a "test configuration". -2. A test configuration is associated with exactly one test suite, and contains descriptions for how to create each of the set of required test resources. +2. A test configuration is associated with exactly one test action (test scenario, test suite, action generator), and contains descriptions for how to create each of the set of required test resources. * The resources required for a particular test definition depend on which test scenarios are included in the test suite. 3. One resource can be used by many different test scenarios. 4. One test scenario may use multiple resources. diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index cd6526cd2f..94db32259b 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -1,21 +1,132 @@ +from __future__ import annotations from typing import Optional, List, Dict from implicitdict import ImplicitDict from monitoring.monitorlib.dicts import JSONAddress +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.reports.validation.definitions import ( ValidationConfiguration, ) from monitoring.uss_qualifier.requirements.definitions import RequirementCollection from monitoring.uss_qualifier.resources.definitions import ResourceCollection +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.suites.definitions import ( TestSuiteActionDeclaration, + TestSuiteTypeName, ) ParticipantID = str """String that refers to a participant being qualified by uss_qualifier""" +class InstanceIndexRange(ImplicitDict): + lo: Optional[int] + """If specified, no indices lower than this value will be included in the range.""" + + i: Optional[int] + """If specified, no index other than this one will be included in the range.""" + + hi: Optional[int] + """If specified, no indices higher than this value will be included in the range.""" + + def includes(self, i: int) -> bool: + if "i" in self and self.i is not None and i != self.i: + return False + if "lo" in self and self.lo is not None and i < self.lo: + return False + if "hi" in self and self.hi is not None and i > self.hi: + return False + return True + + +class ActionGeneratorSelectionCondition(ImplicitDict): + """By default, select all action generators. When specified, limit selection to specified conditions.""" + + types: Optional[List[GeneratorTypeName]] + """Only select action generators of the specified types.""" + + +class TestSuiteSelectionCondition(ImplicitDict): + """By default, select all test suites. When specified, limit selection to specified conditions.""" + + types: Optional[List[TestSuiteTypeName]] + """Only select test suites of the specified types.""" + + +class TestScenarioSelectionCondition(ImplicitDict): + """By default, select all test scenarios. When specified, limit selection to specified conditions.""" + + types: Optional[List[TestScenarioTypeName]] + """Only select test scenarios of the specified types.""" + + +class NthInstanceCondition(ImplicitDict): + """Select an action once a certain number of matching instances have happened.""" + + n: List[InstanceIndexRange] + """Only select an action if it is one of these nth instances.""" + + where_action: TestSuiteActionSelectionCondition + """Condition that an action must meet to be selected as an instance in this condition.""" + + +class AncestorSelectionCondition(ImplicitDict): + """Select ancestor actions meeting all the specified conditions.""" + + of_generation: Optional[int] + """The ancestor is exactly this many generations removed (1 = parent, 2 = grandparent, etc). + + If not specified, an ancestor of any generation meeting the `which` conditions will be selected.""" + + which: List[TestSuiteActionSelectionCondition] + """Only select an ancestor meeting ALL of these conditions.""" + + +class TestSuiteActionSelectionCondition(ImplicitDict): + """Condition for selecting TestSuiteActions. + + If more than one subcondition is specified, satisfaction of ALL subconditions are necessary to select the action.""" + + is_action_generator: Optional[ActionGeneratorSelectionCondition] + """Select these action generator actions.""" + + is_test_suite: Optional[TestSuiteSelectionCondition] + """Select these test suite actions.""" + + is_test_scenario: Optional[TestScenarioSelectionCondition] + """Select these test scenario actions.""" + + regex_matches_name: Optional[str] + """Select actions where this regular expression has a match in the action's name.""" + + defined_at: Optional[List[JSONAddress]] + """Select actions defined at one of the specified addresses. + + The top-level action in a test run is 'test_scenario', 'test_suite', or 'action_generator'. Children use the + 'actions' property, but then must specify the type of the action. So, e.g., the test scenario that is the third + action of a test suite which is the second action in an action generator would be + 'action_generator.actions[1].test_suite.actions[2].test_scenario'. An address that starts or ends with 'actions[i]' + is invalid and will never match.""" + + nth_instance: Optional[NthInstanceCondition] + """Select only certain instances of matching actions.""" + + has_ancestor: Optional[AncestorSelectionCondition] + """Select only actions with a matching ancestor.""" + + except_when: Optional[List[TestSuiteActionSelectionCondition]] + """Do not select actions selected by any of these conditions, even when they are selected by one or more conditions above.""" + + +class ExecutionConfiguration(ImplicitDict): + include_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None + """If specified, only execute test actions if they are selected by ANY of these conditions (and not selected by any of the `skip_when` conditions).""" + + skip_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None + """If specified, do not execute test actions if they are selected by ANY of these conditions.""" + + class TestConfiguration(ImplicitDict): action: TestSuiteActionDeclaration """The action this test configuration wants to run (usually a test suite)""" @@ -26,6 +137,9 @@ class TestConfiguration(ImplicitDict): resources: ResourceCollection """Declarations for resources used by the test suite""" + execution: Optional[ExecutionConfiguration] + """Specification for how to execute the test run.""" + TestedRequirementsCollectionIdentifier = str """Identifier for a requirements collection, local to a TestedRequirementsConfiguration artifact configuration.""" diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml deleted file mode 100644 index acd0084997..0000000000 --- a/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml +++ /dev/null @@ -1,25 +0,0 @@ -v1: - test_run: - resources: - resource_declarations: - che_invalid_flight_intents: {$ref: '../library/resources.yaml#/che_invalid_flight_intents'} - - utm_auth: {$ref: '../library/environment.yaml#/utm_auth'} - uss1_flight_planner: {$ref: '../library/environment.yaml#/uss1_flight_planner'} - scd_dss: {$ref: '../library/environment.yaml#/scd_dss'} - non_baseline_inputs: - - v1.test_run.resources.resource_declarations.utm_auth - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.scd_dss - action: - test_scenario: - scenario_type: scenarios.astm.utm.FlightIntentValidation - resources: - tested_uss: uss1_flight_planner - flight_intents: che_invalid_flight_intents - dss: scd_dss - artifacts: - report: - report_path: output/report_f3548_flight_intent_validation.json - validation: - $ref: ../library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml deleted file mode 100644 index 29dbc20501..0000000000 --- a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml +++ /dev/null @@ -1,28 +0,0 @@ -v1: - test_run: - resources: - resource_declarations: - che_conflicting_flights: {$ref: '../library/resources.yaml#/che_conflicting_flights'} - - utm_auth: {$ref: '../library/environment.yaml#/utm_auth'} - uss1_flight_planner: {$ref: '../library/environment.yaml#/uss1_flight_planner'} - uss2_flight_planner: {$ref: '../library/environment.yaml#/uss2_flight_planner'} - scd_dss: {$ref: '../library/environment.yaml#/scd_dss'} - non_baseline_inputs: - - v1.test_run.resources.resource_declarations.utm_auth - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.scd_dss - action: - test_scenario: - scenario_type: scenarios.astm.utm.ConflictEqualPriorityNotPermitted - resources: - tested_uss: uss1_flight_planner - control_uss: uss2_flight_planner - flight_intents: che_conflicting_flights - dss: scd_dss - artifacts: - report: - report_path: output/report_f3548_nominal_planning_conflict_equal_priority_not_permitted.json - validation: - $ref: ../library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml deleted file mode 100644 index 545ff29535..0000000000 --- a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml +++ /dev/null @@ -1,28 +0,0 @@ -v1: - test_run: - resources: - resource_declarations: - che_conflicting_flights: {$ref: '../library/resources.yaml#/che_conflicting_flights'} - - utm_auth: {$ref: '../library/environment.yaml#/utm_auth'} - uss1_flight_planner: {$ref: '../library/environment.yaml#/uss1_flight_planner'} - uss2_flight_planner: {$ref: '../library/environment.yaml#/uss2_flight_planner'} - scd_dss: {$ref: '../library/environment.yaml#/scd_dss'} - non_baseline_inputs: - - v1.test_run.resources.resource_declarations.utm_auth - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.scd_dss - action: - test_scenario: - scenario_type: scenarios.astm.utm.ConflictHigherPriority - resources: - tested_uss: uss1_flight_planner - control_uss: uss2_flight_planner - flight_intents: che_conflicting_flights - dss: scd_dss - artifacts: - report: - report_path: output/report_f3548_nominal_planning_conflict_higher_priority.json - validation: - $ref: ../library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py index 8351de8b2c..e820b1ecb4 100644 --- a/monitoring/uss_qualifier/main.py +++ b/monitoring/uss_qualifier/main.py @@ -33,7 +33,7 @@ compute_signature, compute_baseline_signature, ) -from monitoring.uss_qualifier.suites.suite import TestSuiteAction +from monitoring.uss_qualifier.suites.suite import TestSuiteAction, ExecutionContext from monitoring.uss_qualifier.validation import validate_config @@ -97,9 +97,10 @@ def execute_test_run(whole_config: USSQualifierConfiguration): environment_signature = compute_signature(environment) logger.info("Instantiating top-level test suite action") + context = ExecutionContext(config.execution if "execution" in config else None) action = TestSuiteAction(config.action, resources) logger.info("Running top-level test suite action") - report = action.run() + report = action.run(context) if report.successful(): logger.info("Final result: SUCCESS") else: diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index 3f3eebbe41..fa9a110d98 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -626,9 +626,6 @@ class SkippedActionReport(ImplicitDict): reason: str """The reason the action was skipped.""" - action_declaration_index: int - """Index of the skipped action in the configured declaration.""" - declaration: TestSuiteActionDeclaration """Full declaration of the action that was skipped.""" diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index eb2f4e7a77..356f771b84 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -404,7 +404,10 @@ def append_notes(new_notes): if "end_time" in report and report.end_time: latest_step_time = report.end_time.datetime - dt_s = round((latest_step_time - report.start_time.datetime).total_seconds()) + if latest_step_time is not None: + dt_s = round((latest_step_time - report.start_time.datetime).total_seconds()) + else: + dt_s = 0 dt_m = math.floor(dt_s / 60) dt_s -= dt_m * 60 padding = "0" if dt_s < 10 else "" @@ -448,7 +451,7 @@ def _skipped_action_of(report: SkippedActionReport) -> ActionNode: raise ValueError( f"Cannot process skipped action for test suite that does not define suite_type nor suite_definition" ) - name = "All scenarios in test suite" + name = "All actions in test suite" elif report.declaration.get_action_type() == ActionType.TestScenario: docs = get_documentation_by_name(report.declaration.test_scenario.scenario_type) return ActionNode( @@ -466,7 +469,7 @@ def _skipped_action_of(report: SkippedActionReport) -> ActionNode: node_type=ActionNodeType.ActionGenerator, children=[], ) - name = f"All scenarios from action generator" + name = f"All actions from action generator" else: raise ValueError( f"Cannot process skipped action of type '{report.declaration.get_action_type()}'" diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html index 4595384515..97c3bad31a 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html @@ -46,6 +46,10 @@ h2 { margin-block-end: 0.1em; } + p { + margin-top: 0.1em; + margin-bottom: 0.1em; + } .sticky_cell_value { position: sticky; top: 40px; diff --git a/monitoring/uss_qualifier/reports/validation/report_validation.py b/monitoring/uss_qualifier/reports/validation/report_validation.py index 6939617071..c420b9e8ed 100644 --- a/monitoring/uss_qualifier/reports/validation/report_validation.py +++ b/monitoring/uss_qualifier/reports/validation/report_validation.py @@ -41,6 +41,8 @@ def _validate_action_full_success( success = success and _validate_action_full_success( a, JSONAddress(context + f".action_generator.actions[{i}]") ) + else: + success = True return success @@ -68,7 +70,7 @@ def _validate_action_no_skipped_actions( ) else: logger.error( - f"No skipped actions not achieved because {context}.test_suite had a skipped action for action index {report.skipped_action.action_declaration_index}: {report.skipped_action.reason}" + f"No skipped actions not achieved because {context} was a skipped action: {report.skipped_action.reason}" ) success = False return success diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index 52280c88f4..f442c336c8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -61,6 +61,7 @@ def __init__( "The Misbehavior Scenario requires at least one DSS instance" ) self._dss = dss_pool.dss_instances[0] + self._injected_tests = [] @property def _rid_version(self) -> RIDVersion: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index 0bbca48d26..3ddd6af00c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -53,6 +53,7 @@ def __init__( self._observers = observers self._evaluation_configuration = evaluation_configuration self._dss_pool = dss_pool + self._injected_tests = [] @property def _rid_version(self) -> RIDVersion: diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py index e69de29bb2..d5bf849f97 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py @@ -0,0 +1,2 @@ +from .configure_locality import ConfigureLocality +from .unconfigure_locality import UnconfigureLocality diff --git a/monitoring/uss_qualifier/suites/definitions.py b/monitoring/uss_qualifier/suites/definitions.py index 21f0b8e8ce..f96c39d4c9 100644 --- a/monitoring/uss_qualifier/suites/definitions.py +++ b/monitoring/uss_qualifier/suites/definitions.py @@ -22,8 +22,11 @@ ) +TestSuiteTypeName = FileReference + + class TestSuiteDeclaration(ImplicitDict): - suite_type: Optional[FileReference] + suite_type: Optional[TestSuiteTypeName] """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" suite_definition: Optional[TestSuiteDefinition] diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index 472128794d..ad5fce1e7e 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -42,6 +42,7 @@ TestSuiteDefinition, ActionType, TestSuiteActionDeclaration, + TestSuiteTypeName, ) @@ -228,7 +229,7 @@ def _render_scenario( def _render_suite_by_type( - suite_type: FileReference, context: TestSuiteRenderContext + suite_type: TestSuiteTypeName, context: TestSuiteRenderContext ) -> List[str]: lines = [] suite_def = ImplicitDict.parse( diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index b356f7e03b..af6726b941 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -1,9 +1,11 @@ from __future__ import annotations import os +from dataclasses import dataclass from datetime import datetime import json -from typing import Dict, List, Optional, Union, Callable, Iterator +import re +from typing import Dict, List, Optional, Union, Iterator import arrow @@ -11,11 +13,19 @@ from loguru import logger import yaml +from monitoring.monitorlib.dicts import JSONAddress from monitoring.monitorlib.inspection import fullname from monitoring.monitorlib.versioning import repo_url_of from monitoring.uss_qualifier.action_generators.action_generator import ( ActionGeneratorType, ActionGenerator, + action_generator_type_from_name, +) +from monitoring.uss_qualifier.configurations.configuration import ( + ExecutionConfiguration, + TestSuiteActionSelectionCondition, + AncestorSelectionCondition, + NthInstanceCondition, ) from monitoring.uss_qualifier.fileio import resolve_filename from monitoring.uss_qualifier.reports.capabilities import ( @@ -96,16 +106,45 @@ def __init__( else: ActionType.raise_invalid_action_declaration() - def run(self) -> TestSuiteActionReport: - if self.test_scenario: - return TestSuiteActionReport(test_scenario=self._run_test_scenario()) - elif self.test_suite: - return TestSuiteActionReport(test_suite=self._run_test_suite()) + def get_name(self) -> str: + if self.test_suite: + return self.test_suite.definition.name elif self.action_generator: - return TestSuiteActionReport(action_generator=self._run_action_generator()) + return self.action_generator.get_name() + elif self.test_scenario: + return self.test_scenario.documentation.name + else: + raise ValueError( + "TestSuiteAction as not a suite, action generator, nor scenario" + ) + + def run(self, context: ExecutionContext) -> TestSuiteActionReport: + context.begin_action(self) + skip_report = context.evaluate_skip() + if skip_report: + logger.warning( + f"Skipping {self.declaration.get_action_type()} '{self.get_name()}' because: {skip_report.reason}" + ) + report = TestSuiteActionReport(skipped_action=skip_report) + else: + if self.test_scenario: + report = TestSuiteActionReport(test_scenario=self._run_test_scenario()) + elif self.test_suite: + report = TestSuiteActionReport(test_suite=self._run_test_suite(context)) + elif self.action_generator: + report = TestSuiteActionReport( + action_generator=self._run_action_generator(context) + ) + else: + raise ValueError( + "TestSuiteAction was not a test scenario, test suite, nor action generator" + ) + context.end_action(self, report) + return report def _run_test_scenario(self) -> TestScenarioReport: scenario = self.test_scenario + logger.info(f'Running "{scenario.documentation.name}" scenario...') scenario.on_failed_check = _print_failed_check try: @@ -134,20 +173,20 @@ def _run_test_scenario(self) -> TestScenarioReport: logger.warning(f'FAILURE for "{scenario.documentation.name}" scenario') return report - def _run_test_suite(self) -> TestSuiteReport: + def _run_test_suite(self, context: ExecutionContext) -> TestSuiteReport: logger.info(f"Beginning test suite {self.test_suite.definition.name}...") - report = self.test_suite.run() + report = self.test_suite.run(context) logger.info(f"Completed test suite {self.test_suite.definition.name}") return report - def _run_action_generator(self) -> ActionGeneratorReport: + def _run_action_generator(self, context: ExecutionContext) -> ActionGeneratorReport: report = ActionGeneratorReport( actions=[], generator_type=self.action_generator.definition.generator_type, start_time=StringBasedDateTime(arrow.utcnow()), ) - _run_actions(self.action_generator.actions(), report) + _run_actions(self.action_generator.actions(), context, report) return report @@ -226,7 +265,6 @@ def __init__( SkippedActionReport( timestamp=StringBasedDateTime(arrow.utcnow().datetime), reason=str(e), - action_declaration_index=a, declaration=action_dec, ) ) @@ -257,7 +295,7 @@ def _make_report_evaluation_action( ) return action - def run(self) -> TestSuiteReport: + def run(self, context: ExecutionContext) -> TestSuiteReport: report = TestSuiteReport( name=self.definition.name, suite_type=self.declaration.type_name, @@ -274,7 +312,7 @@ def actions() -> Iterator[Union[TestSuiteAction, SkippedActionReport]]: if self.definition.has_field_with_value("report_evaluation_scenario"): yield self._make_report_evaluation_action(report) - _run_actions(actions(), report) + _run_actions(actions(), context, report) # Evaluate participants' capabilities if ( @@ -309,6 +347,7 @@ def actions() -> Iterator[Union[TestSuiteAction, SkippedActionReport]]: def _run_actions( actions: Iterator[Union[TestSuiteAction, SkippedActionReport]], + context: ExecutionContext, report: Union[TestSuiteReport, ActionGeneratorReport], ) -> None: success = True @@ -316,7 +355,7 @@ def _run_actions( if isinstance(action, SkippedActionReport): action_report = TestSuiteActionReport(skipped_action=action) else: - action_report = action.run() + action_report = action.run(context) report.actions.append(action_report) if action_report.has_critical_problem(): success = False @@ -333,3 +372,227 @@ def _run_actions( ) report.successful = success report.end_time = StringBasedDateTime(datetime.utcnow()) + + +@dataclass +class ActionStackFrame(object): + action: TestSuiteAction + parent: Optional[ActionStackFrame] + children: List[ActionStackFrame] + + def address(self) -> JSONAddress: + if self.action.test_scenario is not None: + addr = "test_scenario" + elif self.action.test_suite is not None: + addr = "test_suite" + elif self.action.action_generator is not None: + addr = "action_generator" + else: + raise ValueError( + "TestSuiteAction was not a scenario, suite, or action generator" + ) + + if self.parent is None: + return addr + + index = -1 + for a, child in enumerate(self.parent.children): + if child is self: + index = a + break + if index == -1: + raise RuntimeError( + "ActionStackFrame was not listed as a child of its parent" + ) + return f"{self.parent.address()}.actions[{index}].{addr}" + + +class ExecutionContext(object): + config: Optional[ExecutionConfiguration] + top_frame: Optional[ActionStackFrame] + current_frame: Optional[ActionStackFrame] + + def __init__(self, config: Optional[ExecutionConfiguration]): + self.config = config + self.top_frame = None + self.current_frame = None + + def _compute_n_of( + self, target: TestSuiteAction, condition: TestSuiteActionSelectionCondition + ) -> int: + n = 0 + queue = [self.top_frame] + while queue: + frame = queue.pop(0) + if self._is_selected_by(frame, condition): + n += 1 + if frame.action is target: + return n + for c, child in enumerate(frame.children): + queue.insert(c, child) + raise RuntimeError( + f"Could not find target action '{target.get_name()}' anywhere in ExecutionContext" + ) + + def _ancestor_selected_by( + self, + frame: Optional[ActionStackFrame], + of_generation: Optional[int], + which: List[TestSuiteActionSelectionCondition], + ) -> bool: + if frame is None: + return False + + if of_generation is not None: + check_self = of_generation == 0 + of_generation -= 1 + else: + check_self = True + + if check_self: + if all(self._is_selected_by(frame, c) for c in which): + return True + + return self._ancestor_selected_by(frame.parent, of_generation, which) + + def _is_selected_by( + self, frame: ActionStackFrame, f: TestSuiteActionSelectionCondition + ) -> bool: + action = frame.action + result = False + + if "is_action_generator" in f and f.is_action_generator is not None: + if action.action_generator: + if ( + "types" in f.is_action_generator + and f.is_action_generator.types is not None + ): + if not any( + type(action.action_generator) + is action_generator_type_from_name(t) + for t in f.is_action_generator.types + ): + return False + result = True + else: + return False + + if "is_test_suite" in f and f.is_test_suite is not None: + if action.test_suite: + if "types" in f.is_test_suite and f.is_test_suite.types is not None: + if ( + action.test_suite.declaration.suite_type + not in f.is_test_suite.types + ): + return False + result = True + else: + return False + + if "is_test_scenario" in f and f.is_test_scenario is not None: + if action.test_scenario: + if ( + "types" in f.is_test_scenario + and f.is_test_scenario.types is not None + ): + if ( + action.test_scenario.declaration.scenario_type + not in f.is_test_scenario.types + ): + return False + result = True + else: + return False + + if "regex_matches_name" in f and f.regex_matches_name is not None: + if re.search(f.regex_matches_name, action.get_name()) is None: + return False + result = True + + if "defined_at" in f and f.defined_at is not None: + if frame.address() not in f.defined_at: + return False + result = True + + if "nth_instance" in f and f.nth_instance is not None: + if self._is_selected_by(frame, f.nth_instance.where_action): + n = self._compute_n_of(frame.action, f.nth_instance.where_action) + if not any(r.includes(n) for r in f.nth_instance.n): + return False + result = True + else: + return False + + if "has_ancestor" in f and f.has_ancestor is not None: + if ( + "of_generation" in f.has_ancestor + and f.has_ancestor.of_generation is not None + ): + of_generation = f.has_ancestor.of_generation - 1 + else: + of_generation = None + if not self._ancestor_selected_by( + frame.parent, of_generation, f.has_ancestor.which + ): + return False + result = True + + if result and "except_when" in f and f.except_when is not None: + if any(self._is_selected_by(frame, c) for c in f.except_when): + return False + + return result + + def evaluate_skip(self) -> Optional[SkippedActionReport]: + """Decide whether to skip the action in the current_frame or not. + + Should be called in between self.begin_action and self.end_action, and before executing the action. + + Returns: Report regarding skipped action if it should be skipped, otherwise None. + """ + + if not self.config: + return None + + if "include_action_when" in self.config and self.config.include_action_when: + include = False + for condition in self.config.include_action_when: + if self._is_selected_by(self.current_frame, condition): + include = True + break + if not include: + return SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow()), + reason="None of the include_action_when conditions selected the action", + declaration=self.current_frame.action.declaration, + ) + + if "skip_action_when" in self.config and self.config.skip_action_when: + for f, condition in enumerate(self.config.skip_action_when): + if self._is_selected_by(self.current_frame, condition): + return SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow()), + reason=f"Action selected to be skipped by skip_action_when condition {f}", + declaration=self.current_frame.action.declaration, + ) + + return None + + def begin_action(self, action: TestSuiteAction) -> None: + if self.top_frame is None: + self.top_frame = ActionStackFrame(action=action, parent=None, children=[]) + self.current_frame = self.top_frame + else: + self.current_frame = ActionStackFrame( + action=action, parent=self.current_frame, children=[] + ) + self.current_frame.parent.children.append(self.current_frame) + + def end_action( + self, action: TestSuiteAction, report: TestSuiteActionReport + ) -> None: + if self.current_frame.action is not action: + raise RuntimeError( + f"Action {self.current_frame.action.declaration.get_action_type()} {self.current_frame.action.declaration.get_child_type()} was started, but a different action {action.declaration.get_action_type()} {action.declaration.get_child_type()} was ended" + ) + self.current_frame = self.current_frame.parent diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json new file mode 100644 index 0000000000..8f02ae97b0 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json @@ -0,0 +1,22 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "By default, select all action generators. When specified, limit selection to specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.ActionGeneratorSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "types": { + "description": "Only select action generators of the specified types.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json new file mode 100644 index 0000000000..f82effd3ea --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json @@ -0,0 +1,29 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Select ancestor actions meeting all the specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.AncestorSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "of_generation": { + "description": "The ancestor is exactly this many generations removed (1 = parent, 2 = grandparent, etc).\n\nIf not specified, an ancestor of any generation meeting the `which` conditions will be selected.", + "type": [ + "integer", + "null" + ] + }, + "which": { + "description": "Only select an ancestor meeting ALL of these conditions.", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": "array" + } + }, + "required": [ + "which" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json new file mode 100644 index 0000000000..b94efac0a6 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json @@ -0,0 +1,32 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.ExecutionConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "include_action_when": { + "description": "If specified, only execute test actions if they are selected by ANY of these conditions (and not selected by any of the `skip_when` conditions).", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": [ + "array", + "null" + ] + }, + "skip_action_when": { + "description": "If specified, do not execute test actions if they are selected by ANY of these conditions.", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json b/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json new file mode 100644 index 0000000000..758c156c58 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json @@ -0,0 +1,33 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.InstanceIndexRange, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "hi": { + "description": "If specified, no indices higher than this value will be included in the range.", + "type": [ + "integer", + "null" + ] + }, + "i": { + "description": "If specified, no index other than this one will be included in the range.", + "type": [ + "integer", + "null" + ] + }, + "lo": { + "description": "If specified, no indices lower than this value will be included in the range.", + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json new file mode 100644 index 0000000000..60c103a897 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json @@ -0,0 +1,27 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Select an action once a certain number of matching instances have happened.\n\nmonitoring.uss_qualifier.configurations.configuration.NthInstanceCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "n": { + "description": "Only select an action if it is one of these nth instances.", + "items": { + "$ref": "InstanceIndexRange.json" + }, + "type": "array" + }, + "where_action": { + "$ref": "TestSuiteActionSelectionCondition.json", + "description": "Condition that an action must meet to be selected as an instance in this condition." + } + }, + "required": [ + "n", + "where_action" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json index 800958b4e7..7288d30935 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json @@ -11,6 +11,17 @@ "$ref": "../../suites/definitions/TestSuiteActionDeclaration.json", "description": "The action this test configuration wants to run (usually a test suite)" }, + "execution": { + "description": "Specification for how to execute the test run.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "ExecutionConfiguration.json" + } + ] + }, "non_baseline_inputs": { "description": "List of portions of the configuration that should not be considered when computing the test baseline signature (e.g., environmental definitions).", "items": { diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json new file mode 100644 index 0000000000..6c7ec8e3f1 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json @@ -0,0 +1,22 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "By default, select all test scenarios. When specified, limit selection to specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.TestScenarioSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "types": { + "description": "Only select test scenarios of the specified types.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json new file mode 100644 index 0000000000..4e949bbd8a --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json @@ -0,0 +1,94 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Condition for selecting TestSuiteActions.\n\nIf more than one subcondition is specified, satisfaction of ALL subconditions are necessary to select the action.\n\nmonitoring.uss_qualifier.configurations.configuration.TestSuiteActionSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "defined_at": { + "description": "Select actions defined at one of the specified addresses.\n\nThe top-level action in a test run is 'test_scenario', 'test_suite', or 'action_generator'. Children use the\n'actions' property, but then must specify the type of the action. So, e.g., the test scenario that is the third\naction of a test suite which is the second action in an action generator would be\n'action_generator.actions[1].test_suite.actions[2].test_scenario'. An address that starts or ends with 'actions[i]'\nis invalid and will never match.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "except_when": { + "description": "Do not select actions selected by any of these conditions, even when they are selected by one or more conditions above.", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": [ + "array", + "null" + ] + }, + "has_ancestor": { + "description": "Select only actions with a matching ancestor.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "AncestorSelectionCondition.json" + } + ] + }, + "is_action_generator": { + "description": "Select these action generator actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "ActionGeneratorSelectionCondition.json" + } + ] + }, + "is_test_scenario": { + "description": "Select these test scenario actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestScenarioSelectionCondition.json" + } + ] + }, + "is_test_suite": { + "description": "Select these test suite actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestSuiteSelectionCondition.json" + } + ] + }, + "nth_instance": { + "description": "Select only certain instances of matching actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "NthInstanceCondition.json" + } + ] + }, + "regex_matches_name": { + "description": "Select actions where this regular expression has a match in the action's name.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json new file mode 100644 index 0000000000..61cfdd71bb --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json @@ -0,0 +1,22 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "By default, select all test suites. When specified, limit selection to specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.TestSuiteSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "types": { + "description": "Only select test suites of the specified types.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json b/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json index 2d392c57a2..2eec1a0708 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json @@ -7,10 +7,6 @@ "description": "Path to content that replaces the $ref", "type": "string" }, - "action_declaration_index": { - "description": "Index of the skipped action in the configured declaration.", - "type": "integer" - }, "declaration": { "$ref": "../../suites/definitions/TestSuiteActionDeclaration.json", "description": "Full declaration of the action that was skipped." @@ -26,7 +22,6 @@ } }, "required": [ - "action_declaration_index", "declaration", "reason", "timestamp"