From 7fb0ef7d634a751539438f2c5524512b42785e02 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 30 Oct 2023 06:33:12 -0700 Subject: [PATCH 1/4] [uss_qualifier] Add execution control (#292) * Add execution control * Fix behavior with missing execution specification --- .../documentation/definitions.py | 8 +- .../uss_qualifier/configurations/README.md | 135 +++++++- .../configurations/configuration.py | 114 +++++++ .../dev/f3548/flight_intent_validation.yaml | 25 -- ...conflict_equal_priority_not_permitted.yaml | 28 -- ...nal_planning_conflict_higher_priority.yaml | 28 -- monitoring/uss_qualifier/main.py | 5 +- monitoring/uss_qualifier/reports/report.py | 3 - .../uss_qualifier/reports/sequence_view.py | 9 +- .../templates/sequence_view/overview.html | 4 + .../reports/validation/report_validation.py | 4 +- .../astm/netrid/common/misbehavior.py | 1 + .../astm/netrid/common/nominal_behavior.py | 1 + .../utm/flight_intent_validation/__init__.py | 0 .../scenarios/interuss/mock_uss/__init__.py | 2 + .../uss_qualifier/suites/definitions.py | 5 +- .../suites/documentation/documentation.py | 3 +- monitoring/uss_qualifier/suites/suite.py | 293 +++++++++++++++++- .../ActionGeneratorSelectionCondition.json | 22 ++ .../AncestorSelectionCondition.json | 29 ++ .../configuration/ExecutionConfiguration.json | 32 ++ .../configuration/InstanceIndexRange.json | 33 ++ .../configuration/NthInstanceCondition.json | 27 ++ .../configuration/TestConfiguration.json | 11 + .../TestScenarioSelectionCondition.json | 22 ++ .../TestSuiteActionSelectionCondition.json | 94 ++++++ .../TestSuiteSelectionCondition.json | 22 ++ .../reports/report/SkippedActionReport.json | 5 - 28 files changed, 849 insertions(+), 116 deletions(-) delete mode 100644 monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml delete mode 100644 monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml delete mode 100644 monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/__init__.py create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json diff --git a/monitoring/uss_qualifier/action_generators/documentation/definitions.py b/monitoring/uss_qualifier/action_generators/documentation/definitions.py index 33bb951e80..7557421ef4 100644 --- a/monitoring/uss_qualifier/action_generators/documentation/definitions.py +++ b/monitoring/uss_qualifier/action_generators/documentation/definitions.py @@ -4,7 +4,11 @@ from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.fileio import FileReference from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName -from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition +from monitoring.uss_qualifier.suites.definitions import ( + ActionType, + TestSuiteDefinition, + TestSuiteTypeName, +) class PotentialTestScenarioAction(ImplicitDict): @@ -13,7 +17,7 @@ class PotentialTestScenarioAction(ImplicitDict): class PotentialTestSuiteAction(ImplicitDict): - suite_type: Optional[FileReference] + suite_type: Optional[TestSuiteTypeName] """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" suite_definition: Optional[TestSuiteDefinition] diff --git a/monitoring/uss_qualifier/configurations/README.md b/monitoring/uss_qualifier/configurations/README.md index f052adf98d..a0830a3602 100644 --- a/monitoring/uss_qualifier/configurations/README.md +++ b/monitoring/uss_qualifier/configurations/README.md @@ -2,7 +2,7 @@ ## Usage -To execute a test run with uss_qualifier, a uss_qualifier configuration must be provided. This configuration consists of the test suite to run, along with definitions for all resources needed by that test suite, plus information about artifacts that should be generated. See [`USSQualifierConfiguration`](configuration.py) for the exact schema. +To execute a test run with uss_qualifier, a uss_qualifier configuration must be provided. This configuration consists of the test suite to run, along with definitions for all resources needed by that test suite, plus information about artifacts that should be generated. See [`USSQualifierConfiguration`](configuration.py) for the exact schema and [the dev configurations](./dev) for examples. ### Specifying @@ -67,10 +67,141 @@ Loading _q.json_ results in the object: More details may be found in [`fileio.py`](../fileio.py). +## Execution control + +To skip or selectively execute portions of a test run defined by a configuration, populate [the `execution` field of the `TestConfiguration`](configuration.py). This field controls execution of portions of the test run by skipping actions according to specified criteria. When debugging, this feature can be used to selectively execute only a scenario (or set of scenarios) of interest, or exclude a problematic scenario (or set of scenarios) from execution. Some examples are shown below: + +### Skip all test scenarios: + +_Shows test suite / action generator structure_ + +```yaml +execution: + skip_action_when: + - is_test_scenario: {} +``` + +### Skip a particular test suite + +```yaml +execution: + skip_action_when: + - is_test_suite: + types: [suites.astm.netrid.f3411_22a] +``` + +### Only run two kinds of scenarios + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: + types: [scenarios.interuss.mock_uss.configure_locality.ConfigureLocality, scenarios.astm.utm.FlightIntentValidation] +``` + +### Only run the first, ninth, and tenth test scenarios in the test run + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - nth_instance: + n: [{i: 1}, {lo: 9, hi: 10}] + where_action: + is_test_scenario: {} +``` + +### Only run test scenarios with a matching name + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: {} + regex_matches_name: 'ASTM NetRID DSS: Simple ISA' +``` + +### Run everything except two kinds of test suites + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + except_when: + - regex_matches_name: 'ASTM F3411-22a' + - is_test_suite: + types: [suites.astm.utm.f3548_21] + - is_test_scenario: {} +``` + +### Only run the immediate test scenario children of a particular test suite + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: + has_ancestor: + of_generation: 1 + which: + - is_test_suite: {} + regex_matches_name: 'DSS testing for ASTM NetRID F3548-21' +``` + +### Only run test scenarios that are descendants of a particular test suite + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: + has_ancestor: + which: + - is_test_suite: + types: [suites.astm.utm.f3548_21] +``` + +### Only run the third instance of a particular test scenario name + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - nth_instance: + n: [{i: 3}] + where_action: + regex_matches_name: 'Nominal planning: conflict with higher priority' +``` + +### Only run the test scenarios for the second instance of a particular named action generator + +```yaml +execution: + include_action_when: + - is_action_generator: {} + - is_test_suite: {} + - is_test_scenario: {} + has_ancestor: + which: + - nth_instance: + n: [{i: 2}] + where_action: + is_action_generator: {} + regex_matches_name: 'For each appropriate combination of flight planner\(s\)' +``` + ## Design notes 1. Even though all the scenarios, cases, steps and checks are fully defined for a particular test suite, the scenarios require data customized for a particular ecosystem – this data is provided as "test resources" which are created from the specifications in a "test configuration". -2. A test configuration is associated with exactly one test suite, and contains descriptions for how to create each of the set of required test resources. +2. A test configuration is associated with exactly one test action (test scenario, test suite, action generator), and contains descriptions for how to create each of the set of required test resources. * The resources required for a particular test definition depend on which test scenarios are included in the test suite. 3. One resource can be used by many different test scenarios. 4. One test scenario may use multiple resources. diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index cd6526cd2f..94db32259b 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -1,21 +1,132 @@ +from __future__ import annotations from typing import Optional, List, Dict from implicitdict import ImplicitDict from monitoring.monitorlib.dicts import JSONAddress +from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName from monitoring.uss_qualifier.reports.validation.definitions import ( ValidationConfiguration, ) from monitoring.uss_qualifier.requirements.definitions import RequirementCollection from monitoring.uss_qualifier.resources.definitions import ResourceCollection +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.suites.definitions import ( TestSuiteActionDeclaration, + TestSuiteTypeName, ) ParticipantID = str """String that refers to a participant being qualified by uss_qualifier""" +class InstanceIndexRange(ImplicitDict): + lo: Optional[int] + """If specified, no indices lower than this value will be included in the range.""" + + i: Optional[int] + """If specified, no index other than this one will be included in the range.""" + + hi: Optional[int] + """If specified, no indices higher than this value will be included in the range.""" + + def includes(self, i: int) -> bool: + if "i" in self and self.i is not None and i != self.i: + return False + if "lo" in self and self.lo is not None and i < self.lo: + return False + if "hi" in self and self.hi is not None and i > self.hi: + return False + return True + + +class ActionGeneratorSelectionCondition(ImplicitDict): + """By default, select all action generators. When specified, limit selection to specified conditions.""" + + types: Optional[List[GeneratorTypeName]] + """Only select action generators of the specified types.""" + + +class TestSuiteSelectionCondition(ImplicitDict): + """By default, select all test suites. When specified, limit selection to specified conditions.""" + + types: Optional[List[TestSuiteTypeName]] + """Only select test suites of the specified types.""" + + +class TestScenarioSelectionCondition(ImplicitDict): + """By default, select all test scenarios. When specified, limit selection to specified conditions.""" + + types: Optional[List[TestScenarioTypeName]] + """Only select test scenarios of the specified types.""" + + +class NthInstanceCondition(ImplicitDict): + """Select an action once a certain number of matching instances have happened.""" + + n: List[InstanceIndexRange] + """Only select an action if it is one of these nth instances.""" + + where_action: TestSuiteActionSelectionCondition + """Condition that an action must meet to be selected as an instance in this condition.""" + + +class AncestorSelectionCondition(ImplicitDict): + """Select ancestor actions meeting all the specified conditions.""" + + of_generation: Optional[int] + """The ancestor is exactly this many generations removed (1 = parent, 2 = grandparent, etc). + + If not specified, an ancestor of any generation meeting the `which` conditions will be selected.""" + + which: List[TestSuiteActionSelectionCondition] + """Only select an ancestor meeting ALL of these conditions.""" + + +class TestSuiteActionSelectionCondition(ImplicitDict): + """Condition for selecting TestSuiteActions. + + If more than one subcondition is specified, satisfaction of ALL subconditions are necessary to select the action.""" + + is_action_generator: Optional[ActionGeneratorSelectionCondition] + """Select these action generator actions.""" + + is_test_suite: Optional[TestSuiteSelectionCondition] + """Select these test suite actions.""" + + is_test_scenario: Optional[TestScenarioSelectionCondition] + """Select these test scenario actions.""" + + regex_matches_name: Optional[str] + """Select actions where this regular expression has a match in the action's name.""" + + defined_at: Optional[List[JSONAddress]] + """Select actions defined at one of the specified addresses. + + The top-level action in a test run is 'test_scenario', 'test_suite', or 'action_generator'. Children use the + 'actions' property, but then must specify the type of the action. So, e.g., the test scenario that is the third + action of a test suite which is the second action in an action generator would be + 'action_generator.actions[1].test_suite.actions[2].test_scenario'. An address that starts or ends with 'actions[i]' + is invalid and will never match.""" + + nth_instance: Optional[NthInstanceCondition] + """Select only certain instances of matching actions.""" + + has_ancestor: Optional[AncestorSelectionCondition] + """Select only actions with a matching ancestor.""" + + except_when: Optional[List[TestSuiteActionSelectionCondition]] + """Do not select actions selected by any of these conditions, even when they are selected by one or more conditions above.""" + + +class ExecutionConfiguration(ImplicitDict): + include_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None + """If specified, only execute test actions if they are selected by ANY of these conditions (and not selected by any of the `skip_when` conditions).""" + + skip_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None + """If specified, do not execute test actions if they are selected by ANY of these conditions.""" + + class TestConfiguration(ImplicitDict): action: TestSuiteActionDeclaration """The action this test configuration wants to run (usually a test suite)""" @@ -26,6 +137,9 @@ class TestConfiguration(ImplicitDict): resources: ResourceCollection """Declarations for resources used by the test suite""" + execution: Optional[ExecutionConfiguration] + """Specification for how to execute the test run.""" + TestedRequirementsCollectionIdentifier = str """Identifier for a requirements collection, local to a TestedRequirementsConfiguration artifact configuration.""" diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml deleted file mode 100644 index acd0084997..0000000000 --- a/monitoring/uss_qualifier/configurations/dev/f3548/flight_intent_validation.yaml +++ /dev/null @@ -1,25 +0,0 @@ -v1: - test_run: - resources: - resource_declarations: - che_invalid_flight_intents: {$ref: '../library/resources.yaml#/che_invalid_flight_intents'} - - utm_auth: {$ref: '../library/environment.yaml#/utm_auth'} - uss1_flight_planner: {$ref: '../library/environment.yaml#/uss1_flight_planner'} - scd_dss: {$ref: '../library/environment.yaml#/scd_dss'} - non_baseline_inputs: - - v1.test_run.resources.resource_declarations.utm_auth - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.scd_dss - action: - test_scenario: - scenario_type: scenarios.astm.utm.FlightIntentValidation - resources: - tested_uss: uss1_flight_planner - flight_intents: che_invalid_flight_intents - dss: scd_dss - artifacts: - report: - report_path: output/report_f3548_flight_intent_validation.json - validation: - $ref: ../library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml deleted file mode 100644 index 29dbc20501..0000000000 --- a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_equal_priority_not_permitted.yaml +++ /dev/null @@ -1,28 +0,0 @@ -v1: - test_run: - resources: - resource_declarations: - che_conflicting_flights: {$ref: '../library/resources.yaml#/che_conflicting_flights'} - - utm_auth: {$ref: '../library/environment.yaml#/utm_auth'} - uss1_flight_planner: {$ref: '../library/environment.yaml#/uss1_flight_planner'} - uss2_flight_planner: {$ref: '../library/environment.yaml#/uss2_flight_planner'} - scd_dss: {$ref: '../library/environment.yaml#/scd_dss'} - non_baseline_inputs: - - v1.test_run.resources.resource_declarations.utm_auth - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.scd_dss - action: - test_scenario: - scenario_type: scenarios.astm.utm.ConflictEqualPriorityNotPermitted - resources: - tested_uss: uss1_flight_planner - control_uss: uss2_flight_planner - flight_intents: che_conflicting_flights - dss: scd_dss - artifacts: - report: - report_path: output/report_f3548_nominal_planning_conflict_equal_priority_not_permitted.json - validation: - $ref: ../library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml b/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml deleted file mode 100644 index 545ff29535..0000000000 --- a/monitoring/uss_qualifier/configurations/dev/f3548/nominal_planning_conflict_higher_priority.yaml +++ /dev/null @@ -1,28 +0,0 @@ -v1: - test_run: - resources: - resource_declarations: - che_conflicting_flights: {$ref: '../library/resources.yaml#/che_conflicting_flights'} - - utm_auth: {$ref: '../library/environment.yaml#/utm_auth'} - uss1_flight_planner: {$ref: '../library/environment.yaml#/uss1_flight_planner'} - uss2_flight_planner: {$ref: '../library/environment.yaml#/uss2_flight_planner'} - scd_dss: {$ref: '../library/environment.yaml#/scd_dss'} - non_baseline_inputs: - - v1.test_run.resources.resource_declarations.utm_auth - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.uss1_flight_planner - - v1.test_run.resources.resource_declarations.scd_dss - action: - test_scenario: - scenario_type: scenarios.astm.utm.ConflictHigherPriority - resources: - tested_uss: uss1_flight_planner - control_uss: uss2_flight_planner - flight_intents: che_conflicting_flights - dss: scd_dss - artifacts: - report: - report_path: output/report_f3548_nominal_planning_conflict_higher_priority.json - validation: - $ref: ../library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py index 8351de8b2c..e820b1ecb4 100644 --- a/monitoring/uss_qualifier/main.py +++ b/monitoring/uss_qualifier/main.py @@ -33,7 +33,7 @@ compute_signature, compute_baseline_signature, ) -from monitoring.uss_qualifier.suites.suite import TestSuiteAction +from monitoring.uss_qualifier.suites.suite import TestSuiteAction, ExecutionContext from monitoring.uss_qualifier.validation import validate_config @@ -97,9 +97,10 @@ def execute_test_run(whole_config: USSQualifierConfiguration): environment_signature = compute_signature(environment) logger.info("Instantiating top-level test suite action") + context = ExecutionContext(config.execution if "execution" in config else None) action = TestSuiteAction(config.action, resources) logger.info("Running top-level test suite action") - report = action.run() + report = action.run(context) if report.successful(): logger.info("Final result: SUCCESS") else: diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index 3f3eebbe41..fa9a110d98 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -626,9 +626,6 @@ class SkippedActionReport(ImplicitDict): reason: str """The reason the action was skipped.""" - action_declaration_index: int - """Index of the skipped action in the configured declaration.""" - declaration: TestSuiteActionDeclaration """Full declaration of the action that was skipped.""" diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index eb2f4e7a77..356f771b84 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -404,7 +404,10 @@ def append_notes(new_notes): if "end_time" in report and report.end_time: latest_step_time = report.end_time.datetime - dt_s = round((latest_step_time - report.start_time.datetime).total_seconds()) + if latest_step_time is not None: + dt_s = round((latest_step_time - report.start_time.datetime).total_seconds()) + else: + dt_s = 0 dt_m = math.floor(dt_s / 60) dt_s -= dt_m * 60 padding = "0" if dt_s < 10 else "" @@ -448,7 +451,7 @@ def _skipped_action_of(report: SkippedActionReport) -> ActionNode: raise ValueError( f"Cannot process skipped action for test suite that does not define suite_type nor suite_definition" ) - name = "All scenarios in test suite" + name = "All actions in test suite" elif report.declaration.get_action_type() == ActionType.TestScenario: docs = get_documentation_by_name(report.declaration.test_scenario.scenario_type) return ActionNode( @@ -466,7 +469,7 @@ def _skipped_action_of(report: SkippedActionReport) -> ActionNode: node_type=ActionNodeType.ActionGenerator, children=[], ) - name = f"All scenarios from action generator" + name = f"All actions from action generator" else: raise ValueError( f"Cannot process skipped action of type '{report.declaration.get_action_type()}'" diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html index 4595384515..97c3bad31a 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html @@ -46,6 +46,10 @@ h2 { margin-block-end: 0.1em; } + p { + margin-top: 0.1em; + margin-bottom: 0.1em; + } .sticky_cell_value { position: sticky; top: 40px; diff --git a/monitoring/uss_qualifier/reports/validation/report_validation.py b/monitoring/uss_qualifier/reports/validation/report_validation.py index 6939617071..c420b9e8ed 100644 --- a/monitoring/uss_qualifier/reports/validation/report_validation.py +++ b/monitoring/uss_qualifier/reports/validation/report_validation.py @@ -41,6 +41,8 @@ def _validate_action_full_success( success = success and _validate_action_full_success( a, JSONAddress(context + f".action_generator.actions[{i}]") ) + else: + success = True return success @@ -68,7 +70,7 @@ def _validate_action_no_skipped_actions( ) else: logger.error( - f"No skipped actions not achieved because {context}.test_suite had a skipped action for action index {report.skipped_action.action_declaration_index}: {report.skipped_action.reason}" + f"No skipped actions not achieved because {context} was a skipped action: {report.skipped_action.reason}" ) success = False return success diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index 52280c88f4..f442c336c8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -61,6 +61,7 @@ def __init__( "The Misbehavior Scenario requires at least one DSS instance" ) self._dss = dss_pool.dss_instances[0] + self._injected_tests = [] @property def _rid_version(self) -> RIDVersion: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index 0bbca48d26..3ddd6af00c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -53,6 +53,7 @@ def __init__( self._observers = observers self._evaluation_configuration = evaluation_configuration self._dss_pool = dss_pool + self._injected_tests = [] @property def _rid_version(self) -> RIDVersion: diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py index e69de29bb2..d5bf849f97 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py @@ -0,0 +1,2 @@ +from .configure_locality import ConfigureLocality +from .unconfigure_locality import UnconfigureLocality diff --git a/monitoring/uss_qualifier/suites/definitions.py b/monitoring/uss_qualifier/suites/definitions.py index 21f0b8e8ce..f96c39d4c9 100644 --- a/monitoring/uss_qualifier/suites/definitions.py +++ b/monitoring/uss_qualifier/suites/definitions.py @@ -22,8 +22,11 @@ ) +TestSuiteTypeName = FileReference + + class TestSuiteDeclaration(ImplicitDict): - suite_type: Optional[FileReference] + suite_type: Optional[TestSuiteTypeName] """Type/location of test suite. Usually expressed as the file name of the suite definition (without extension) qualified relative to the `uss_qualifier` folder""" suite_definition: Optional[TestSuiteDefinition] diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index 472128794d..ad5fce1e7e 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -42,6 +42,7 @@ TestSuiteDefinition, ActionType, TestSuiteActionDeclaration, + TestSuiteTypeName, ) @@ -228,7 +229,7 @@ def _render_scenario( def _render_suite_by_type( - suite_type: FileReference, context: TestSuiteRenderContext + suite_type: TestSuiteTypeName, context: TestSuiteRenderContext ) -> List[str]: lines = [] suite_def = ImplicitDict.parse( diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index b356f7e03b..af6726b941 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -1,9 +1,11 @@ from __future__ import annotations import os +from dataclasses import dataclass from datetime import datetime import json -from typing import Dict, List, Optional, Union, Callable, Iterator +import re +from typing import Dict, List, Optional, Union, Iterator import arrow @@ -11,11 +13,19 @@ from loguru import logger import yaml +from monitoring.monitorlib.dicts import JSONAddress from monitoring.monitorlib.inspection import fullname from monitoring.monitorlib.versioning import repo_url_of from monitoring.uss_qualifier.action_generators.action_generator import ( ActionGeneratorType, ActionGenerator, + action_generator_type_from_name, +) +from monitoring.uss_qualifier.configurations.configuration import ( + ExecutionConfiguration, + TestSuiteActionSelectionCondition, + AncestorSelectionCondition, + NthInstanceCondition, ) from monitoring.uss_qualifier.fileio import resolve_filename from monitoring.uss_qualifier.reports.capabilities import ( @@ -96,16 +106,45 @@ def __init__( else: ActionType.raise_invalid_action_declaration() - def run(self) -> TestSuiteActionReport: - if self.test_scenario: - return TestSuiteActionReport(test_scenario=self._run_test_scenario()) - elif self.test_suite: - return TestSuiteActionReport(test_suite=self._run_test_suite()) + def get_name(self) -> str: + if self.test_suite: + return self.test_suite.definition.name elif self.action_generator: - return TestSuiteActionReport(action_generator=self._run_action_generator()) + return self.action_generator.get_name() + elif self.test_scenario: + return self.test_scenario.documentation.name + else: + raise ValueError( + "TestSuiteAction as not a suite, action generator, nor scenario" + ) + + def run(self, context: ExecutionContext) -> TestSuiteActionReport: + context.begin_action(self) + skip_report = context.evaluate_skip() + if skip_report: + logger.warning( + f"Skipping {self.declaration.get_action_type()} '{self.get_name()}' because: {skip_report.reason}" + ) + report = TestSuiteActionReport(skipped_action=skip_report) + else: + if self.test_scenario: + report = TestSuiteActionReport(test_scenario=self._run_test_scenario()) + elif self.test_suite: + report = TestSuiteActionReport(test_suite=self._run_test_suite(context)) + elif self.action_generator: + report = TestSuiteActionReport( + action_generator=self._run_action_generator(context) + ) + else: + raise ValueError( + "TestSuiteAction was not a test scenario, test suite, nor action generator" + ) + context.end_action(self, report) + return report def _run_test_scenario(self) -> TestScenarioReport: scenario = self.test_scenario + logger.info(f'Running "{scenario.documentation.name}" scenario...') scenario.on_failed_check = _print_failed_check try: @@ -134,20 +173,20 @@ def _run_test_scenario(self) -> TestScenarioReport: logger.warning(f'FAILURE for "{scenario.documentation.name}" scenario') return report - def _run_test_suite(self) -> TestSuiteReport: + def _run_test_suite(self, context: ExecutionContext) -> TestSuiteReport: logger.info(f"Beginning test suite {self.test_suite.definition.name}...") - report = self.test_suite.run() + report = self.test_suite.run(context) logger.info(f"Completed test suite {self.test_suite.definition.name}") return report - def _run_action_generator(self) -> ActionGeneratorReport: + def _run_action_generator(self, context: ExecutionContext) -> ActionGeneratorReport: report = ActionGeneratorReport( actions=[], generator_type=self.action_generator.definition.generator_type, start_time=StringBasedDateTime(arrow.utcnow()), ) - _run_actions(self.action_generator.actions(), report) + _run_actions(self.action_generator.actions(), context, report) return report @@ -226,7 +265,6 @@ def __init__( SkippedActionReport( timestamp=StringBasedDateTime(arrow.utcnow().datetime), reason=str(e), - action_declaration_index=a, declaration=action_dec, ) ) @@ -257,7 +295,7 @@ def _make_report_evaluation_action( ) return action - def run(self) -> TestSuiteReport: + def run(self, context: ExecutionContext) -> TestSuiteReport: report = TestSuiteReport( name=self.definition.name, suite_type=self.declaration.type_name, @@ -274,7 +312,7 @@ def actions() -> Iterator[Union[TestSuiteAction, SkippedActionReport]]: if self.definition.has_field_with_value("report_evaluation_scenario"): yield self._make_report_evaluation_action(report) - _run_actions(actions(), report) + _run_actions(actions(), context, report) # Evaluate participants' capabilities if ( @@ -309,6 +347,7 @@ def actions() -> Iterator[Union[TestSuiteAction, SkippedActionReport]]: def _run_actions( actions: Iterator[Union[TestSuiteAction, SkippedActionReport]], + context: ExecutionContext, report: Union[TestSuiteReport, ActionGeneratorReport], ) -> None: success = True @@ -316,7 +355,7 @@ def _run_actions( if isinstance(action, SkippedActionReport): action_report = TestSuiteActionReport(skipped_action=action) else: - action_report = action.run() + action_report = action.run(context) report.actions.append(action_report) if action_report.has_critical_problem(): success = False @@ -333,3 +372,227 @@ def _run_actions( ) report.successful = success report.end_time = StringBasedDateTime(datetime.utcnow()) + + +@dataclass +class ActionStackFrame(object): + action: TestSuiteAction + parent: Optional[ActionStackFrame] + children: List[ActionStackFrame] + + def address(self) -> JSONAddress: + if self.action.test_scenario is not None: + addr = "test_scenario" + elif self.action.test_suite is not None: + addr = "test_suite" + elif self.action.action_generator is not None: + addr = "action_generator" + else: + raise ValueError( + "TestSuiteAction was not a scenario, suite, or action generator" + ) + + if self.parent is None: + return addr + + index = -1 + for a, child in enumerate(self.parent.children): + if child is self: + index = a + break + if index == -1: + raise RuntimeError( + "ActionStackFrame was not listed as a child of its parent" + ) + return f"{self.parent.address()}.actions[{index}].{addr}" + + +class ExecutionContext(object): + config: Optional[ExecutionConfiguration] + top_frame: Optional[ActionStackFrame] + current_frame: Optional[ActionStackFrame] + + def __init__(self, config: Optional[ExecutionConfiguration]): + self.config = config + self.top_frame = None + self.current_frame = None + + def _compute_n_of( + self, target: TestSuiteAction, condition: TestSuiteActionSelectionCondition + ) -> int: + n = 0 + queue = [self.top_frame] + while queue: + frame = queue.pop(0) + if self._is_selected_by(frame, condition): + n += 1 + if frame.action is target: + return n + for c, child in enumerate(frame.children): + queue.insert(c, child) + raise RuntimeError( + f"Could not find target action '{target.get_name()}' anywhere in ExecutionContext" + ) + + def _ancestor_selected_by( + self, + frame: Optional[ActionStackFrame], + of_generation: Optional[int], + which: List[TestSuiteActionSelectionCondition], + ) -> bool: + if frame is None: + return False + + if of_generation is not None: + check_self = of_generation == 0 + of_generation -= 1 + else: + check_self = True + + if check_self: + if all(self._is_selected_by(frame, c) for c in which): + return True + + return self._ancestor_selected_by(frame.parent, of_generation, which) + + def _is_selected_by( + self, frame: ActionStackFrame, f: TestSuiteActionSelectionCondition + ) -> bool: + action = frame.action + result = False + + if "is_action_generator" in f and f.is_action_generator is not None: + if action.action_generator: + if ( + "types" in f.is_action_generator + and f.is_action_generator.types is not None + ): + if not any( + type(action.action_generator) + is action_generator_type_from_name(t) + for t in f.is_action_generator.types + ): + return False + result = True + else: + return False + + if "is_test_suite" in f and f.is_test_suite is not None: + if action.test_suite: + if "types" in f.is_test_suite and f.is_test_suite.types is not None: + if ( + action.test_suite.declaration.suite_type + not in f.is_test_suite.types + ): + return False + result = True + else: + return False + + if "is_test_scenario" in f and f.is_test_scenario is not None: + if action.test_scenario: + if ( + "types" in f.is_test_scenario + and f.is_test_scenario.types is not None + ): + if ( + action.test_scenario.declaration.scenario_type + not in f.is_test_scenario.types + ): + return False + result = True + else: + return False + + if "regex_matches_name" in f and f.regex_matches_name is not None: + if re.search(f.regex_matches_name, action.get_name()) is None: + return False + result = True + + if "defined_at" in f and f.defined_at is not None: + if frame.address() not in f.defined_at: + return False + result = True + + if "nth_instance" in f and f.nth_instance is not None: + if self._is_selected_by(frame, f.nth_instance.where_action): + n = self._compute_n_of(frame.action, f.nth_instance.where_action) + if not any(r.includes(n) for r in f.nth_instance.n): + return False + result = True + else: + return False + + if "has_ancestor" in f and f.has_ancestor is not None: + if ( + "of_generation" in f.has_ancestor + and f.has_ancestor.of_generation is not None + ): + of_generation = f.has_ancestor.of_generation - 1 + else: + of_generation = None + if not self._ancestor_selected_by( + frame.parent, of_generation, f.has_ancestor.which + ): + return False + result = True + + if result and "except_when" in f and f.except_when is not None: + if any(self._is_selected_by(frame, c) for c in f.except_when): + return False + + return result + + def evaluate_skip(self) -> Optional[SkippedActionReport]: + """Decide whether to skip the action in the current_frame or not. + + Should be called in between self.begin_action and self.end_action, and before executing the action. + + Returns: Report regarding skipped action if it should be skipped, otherwise None. + """ + + if not self.config: + return None + + if "include_action_when" in self.config and self.config.include_action_when: + include = False + for condition in self.config.include_action_when: + if self._is_selected_by(self.current_frame, condition): + include = True + break + if not include: + return SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow()), + reason="None of the include_action_when conditions selected the action", + declaration=self.current_frame.action.declaration, + ) + + if "skip_action_when" in self.config and self.config.skip_action_when: + for f, condition in enumerate(self.config.skip_action_when): + if self._is_selected_by(self.current_frame, condition): + return SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow()), + reason=f"Action selected to be skipped by skip_action_when condition {f}", + declaration=self.current_frame.action.declaration, + ) + + return None + + def begin_action(self, action: TestSuiteAction) -> None: + if self.top_frame is None: + self.top_frame = ActionStackFrame(action=action, parent=None, children=[]) + self.current_frame = self.top_frame + else: + self.current_frame = ActionStackFrame( + action=action, parent=self.current_frame, children=[] + ) + self.current_frame.parent.children.append(self.current_frame) + + def end_action( + self, action: TestSuiteAction, report: TestSuiteActionReport + ) -> None: + if self.current_frame.action is not action: + raise RuntimeError( + f"Action {self.current_frame.action.declaration.get_action_type()} {self.current_frame.action.declaration.get_child_type()} was started, but a different action {action.declaration.get_action_type()} {action.declaration.get_child_type()} was ended" + ) + self.current_frame = self.current_frame.parent diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json new file mode 100644 index 0000000000..8f02ae97b0 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json @@ -0,0 +1,22 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ActionGeneratorSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "By default, select all action generators. When specified, limit selection to specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.ActionGeneratorSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "types": { + "description": "Only select action generators of the specified types.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json new file mode 100644 index 0000000000..f82effd3ea --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json @@ -0,0 +1,29 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/AncestorSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Select ancestor actions meeting all the specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.AncestorSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "of_generation": { + "description": "The ancestor is exactly this many generations removed (1 = parent, 2 = grandparent, etc).\n\nIf not specified, an ancestor of any generation meeting the `which` conditions will be selected.", + "type": [ + "integer", + "null" + ] + }, + "which": { + "description": "Only select an ancestor meeting ALL of these conditions.", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": "array" + } + }, + "required": [ + "which" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json new file mode 100644 index 0000000000..b94efac0a6 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json @@ -0,0 +1,32 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.ExecutionConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "include_action_when": { + "description": "If specified, only execute test actions if they are selected by ANY of these conditions (and not selected by any of the `skip_when` conditions).", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": [ + "array", + "null" + ] + }, + "skip_action_when": { + "description": "If specified, do not execute test actions if they are selected by ANY of these conditions.", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json b/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json new file mode 100644 index 0000000000..758c156c58 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json @@ -0,0 +1,33 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/InstanceIndexRange.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.InstanceIndexRange, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "hi": { + "description": "If specified, no indices higher than this value will be included in the range.", + "type": [ + "integer", + "null" + ] + }, + "i": { + "description": "If specified, no index other than this one will be included in the range.", + "type": [ + "integer", + "null" + ] + }, + "lo": { + "description": "If specified, no indices lower than this value will be included in the range.", + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json new file mode 100644 index 0000000000..60c103a897 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json @@ -0,0 +1,27 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/NthInstanceCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Select an action once a certain number of matching instances have happened.\n\nmonitoring.uss_qualifier.configurations.configuration.NthInstanceCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "n": { + "description": "Only select an action if it is one of these nth instances.", + "items": { + "$ref": "InstanceIndexRange.json" + }, + "type": "array" + }, + "where_action": { + "$ref": "TestSuiteActionSelectionCondition.json", + "description": "Condition that an action must meet to be selected as an instance in this condition." + } + }, + "required": [ + "n", + "where_action" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json index 800958b4e7..7288d30935 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestConfiguration.json @@ -11,6 +11,17 @@ "$ref": "../../suites/definitions/TestSuiteActionDeclaration.json", "description": "The action this test configuration wants to run (usually a test suite)" }, + "execution": { + "description": "Specification for how to execute the test run.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "ExecutionConfiguration.json" + } + ] + }, "non_baseline_inputs": { "description": "List of portions of the configuration that should not be considered when computing the test baseline signature (e.g., environmental definitions).", "items": { diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json new file mode 100644 index 0000000000..6c7ec8e3f1 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json @@ -0,0 +1,22 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestScenarioSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "By default, select all test scenarios. When specified, limit selection to specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.TestScenarioSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "types": { + "description": "Only select test scenarios of the specified types.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json new file mode 100644 index 0000000000..4e949bbd8a --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json @@ -0,0 +1,94 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteActionSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Condition for selecting TestSuiteActions.\n\nIf more than one subcondition is specified, satisfaction of ALL subconditions are necessary to select the action.\n\nmonitoring.uss_qualifier.configurations.configuration.TestSuiteActionSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "defined_at": { + "description": "Select actions defined at one of the specified addresses.\n\nThe top-level action in a test run is 'test_scenario', 'test_suite', or 'action_generator'. Children use the\n'actions' property, but then must specify the type of the action. So, e.g., the test scenario that is the third\naction of a test suite which is the second action in an action generator would be\n'action_generator.actions[1].test_suite.actions[2].test_scenario'. An address that starts or ends with 'actions[i]'\nis invalid and will never match.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "except_when": { + "description": "Do not select actions selected by any of these conditions, even when they are selected by one or more conditions above.", + "items": { + "$ref": "TestSuiteActionSelectionCondition.json" + }, + "type": [ + "array", + "null" + ] + }, + "has_ancestor": { + "description": "Select only actions with a matching ancestor.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "AncestorSelectionCondition.json" + } + ] + }, + "is_action_generator": { + "description": "Select these action generator actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "ActionGeneratorSelectionCondition.json" + } + ] + }, + "is_test_scenario": { + "description": "Select these test scenario actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestScenarioSelectionCondition.json" + } + ] + }, + "is_test_suite": { + "description": "Select these test suite actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "TestSuiteSelectionCondition.json" + } + ] + }, + "nth_instance": { + "description": "Select only certain instances of matching actions.", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "NthInstanceCondition.json" + } + ] + }, + "regex_matches_name": { + "description": "Select actions where this regular expression has a match in the action's name.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json new file mode 100644 index 0000000000..61cfdd71bb --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json @@ -0,0 +1,22 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/TestSuiteSelectionCondition.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "By default, select all test suites. When specified, limit selection to specified conditions.\n\nmonitoring.uss_qualifier.configurations.configuration.TestSuiteSelectionCondition, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "types": { + "description": "Only select test suites of the specified types.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json b/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json index 2d392c57a2..2eec1a0708 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/SkippedActionReport.json @@ -7,10 +7,6 @@ "description": "Path to content that replaces the $ref", "type": "string" }, - "action_declaration_index": { - "description": "Index of the skipped action in the configured declaration.", - "type": "integer" - }, "declaration": { "$ref": "../../suites/definitions/TestSuiteActionDeclaration.json", "description": "Full declaration of the action that was skipped." @@ -26,7 +22,6 @@ } }, "required": [ - "action_declaration_index", "declaration", "reason", "timestamp" From 345ff7c4acc40d9092a6bef0f5646a0a78c36b72 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Tue, 31 Oct 2023 08:53:38 -0700 Subject: [PATCH 2/4] [mock_uss] Reorganize mock_uss flight functionality (#294) Reorganize mock_uss flight functionality --- monitoring/mock_uss/README.md | 5 +- monitoring/mock_uss/__init__.py | 4 +- monitoring/mock_uss/f3548v21/README.md | 3 + .../mock_uss/{scdsc => f3548v21}/__init__.py | 0 .../{scdsc => f3548v21}/flight_planning.py | 34 ++++++++--- .../routes_scd.py} | 33 +--------- monitoring/mock_uss/flight_planning/README.md | 3 + monitoring/mock_uss/flight_planning/routes.py | 2 +- monitoring/mock_uss/flights/README.md | 3 + monitoring/mock_uss/flights/__init__.py | 0 .../mock_uss/{scdsc => flights}/database.py | 3 + monitoring/mock_uss/flights/planning.py | 47 ++++++++++++++ monitoring/mock_uss/scd_injection/README.md | 3 + monitoring/mock_uss/scd_injection/__init__.py | 0 monitoring/mock_uss/scd_injection/routes.py | 6 ++ .../routes_injection.py | 61 ++++--------------- monitoring/mock_uss/scdsc/README.md | 1 - monitoring/mock_uss/scdsc/routes.py | 10 --- monitoring/mock_uss/uspace/README.md | 3 + monitoring/mock_uss/uspace/__init__.py | 0 monitoring/mock_uss/uspace/flight_auth.py | 14 +++++ 21 files changed, 131 insertions(+), 104 deletions(-) create mode 100644 monitoring/mock_uss/f3548v21/README.md rename monitoring/mock_uss/{scdsc => f3548v21}/__init__.py (100%) rename monitoring/mock_uss/{scdsc => f3548v21}/flight_planning.py (88%) rename monitoring/mock_uss/{scdsc/routes_scdsc.py => f3548v21/routes_scd.py} (66%) create mode 100644 monitoring/mock_uss/flight_planning/README.md create mode 100644 monitoring/mock_uss/flights/README.md create mode 100644 monitoring/mock_uss/flights/__init__.py rename monitoring/mock_uss/{scdsc => flights}/database.py (93%) create mode 100644 monitoring/mock_uss/flights/planning.py create mode 100644 monitoring/mock_uss/scd_injection/README.md create mode 100644 monitoring/mock_uss/scd_injection/__init__.py create mode 100644 monitoring/mock_uss/scd_injection/routes.py rename monitoring/mock_uss/{scdsc => scd_injection}/routes_injection.py (91%) delete mode 100644 monitoring/mock_uss/scdsc/README.md delete mode 100644 monitoring/mock_uss/scdsc/routes.py create mode 100644 monitoring/mock_uss/uspace/README.md create mode 100644 monitoring/mock_uss/uspace/__init__.py create mode 100644 monitoring/mock_uss/uspace/flight_auth.py diff --git a/monitoring/mock_uss/README.md b/monitoring/mock_uss/README.md index 91488cc899..f770ba3e0e 100644 --- a/monitoring/mock_uss/README.md +++ b/monitoring/mock_uss/README.md @@ -18,9 +18,10 @@ The available functionality sets are: * [`msgsigning`](msgsigning): [IETF HTTP Message Signatures](https://datatracker.ietf.org/doc/draft-ietf-httpbis-message-signatures/) * [`riddp`](riddp): Remote ID Display Provider * [`ridsp`](ridsp): Remote ID Service Provider -* [`scdsc`](scdsc): ASTM F3548 strategic coordinator +* `scdsc`: Combination of [ASTM F3548-21](f3548v21) strategic conflict detection and [scd flight injection](scd_injection) +* [`flight_planning`](flight_planning): Exposes [InterUSS flight_planning automated testing API](https://github.com/interuss/automated_testing_interfaces/tree/main/flight_planning) * [`tracer`](tracer): Interoperability ecosystem tracer logger -* [`interaction_logging`](interaction_logging): Enables logging of the [interuss](https://github.com/astm-utm/Protocol/blob/master/utm.yaml) interactions between mock_uss and other uss participants +* [`interaction_logging`](interaction_logging): Enables logging of interactions between mock_uss and other uss participants ## Local deployment diff --git a/monitoring/mock_uss/__init__.py b/monitoring/mock_uss/__init__.py index 7733a0829b..4aa3c4ab35 100644 --- a/monitoring/mock_uss/__init__.py +++ b/monitoring/mock_uss/__init__.py @@ -81,8 +81,8 @@ def require_config_value(config_key: str) -> None: if SERVICE_SCDSC in webapp.config[config.KEY_SERVICES]: enabled_services.add(SERVICE_SCDSC) - from monitoring.mock_uss import scdsc - from monitoring.mock_uss.scdsc import routes as scdsc_routes + from monitoring.mock_uss.f3548v21 import routes_scd + from monitoring.mock_uss.scd_injection import routes as scd_injection_routes if SERVICE_MESSAGESIGNING in webapp.config[config.KEY_SERVICES]: enabled_services.add(SERVICE_MESSAGESIGNING) diff --git a/monitoring/mock_uss/f3548v21/README.md b/monitoring/mock_uss/f3548v21/README.md new file mode 100644 index 0000000000..9f7137ca84 --- /dev/null +++ b/monitoring/mock_uss/f3548v21/README.md @@ -0,0 +1,3 @@ +# mock_uss: ASTM F3548-21 + +[ASTM F3548-21](http://astm.org/f3548-21.html) standardizes UTM interoperability between USSs to achieve strategic coordination and communicate constraints. This folder enables [mock_uss](..) to comply with the Strategic Conflict Detection requirements from that standard. diff --git a/monitoring/mock_uss/scdsc/__init__.py b/monitoring/mock_uss/f3548v21/__init__.py similarity index 100% rename from monitoring/mock_uss/scdsc/__init__.py rename to monitoring/mock_uss/f3548v21/__init__.py diff --git a/monitoring/mock_uss/scdsc/flight_planning.py b/monitoring/mock_uss/f3548v21/flight_planning.py similarity index 88% rename from monitoring/mock_uss/scdsc/flight_planning.py rename to monitoring/mock_uss/f3548v21/flight_planning.py index d0c9457fa9..c477ce1067 100644 --- a/monitoring/mock_uss/scdsc/flight_planning.py +++ b/monitoring/mock_uss/f3548v21/flight_planning.py @@ -2,14 +2,16 @@ from typing import Optional, List, Callable import arrow + +from monitoring.uss_qualifier.resources.overrides import apply_overrides from uas_standards.astm.f3548.v21 import api as f3548_v21 +from uas_standards.astm.f3548.v21.api import OperationalIntentDetails, OperationalIntent from uas_standards.astm.f3548.v21.constants import OiMaxVertices, OiMaxPlanHorizonDays from uas_standards.interuss.automated_testing.scd.v1 import api as scd_api -from monitoring.mock_uss.scdsc.database import FlightRecord +from monitoring.mock_uss.flights.database import FlightRecord from monitoring.monitorlib.geotemporal import Volume4DCollection from monitoring.monitorlib.locality import Locality -from monitoring.monitorlib.uspace import problems_with_flight_authorisation from uas_standards.interuss.automated_testing.scd.v1.api import OperationalIntentState @@ -17,19 +19,13 @@ class PlanningError(Exception): pass -def validate_request(req_body: scd_api.InjectFlightRequest, locality: Locality) -> None: +def validate_request(req_body: scd_api.InjectFlightRequest) -> None: """Raise a PlannerError if the request is not valid. Args: req_body: Information about the requested flight. locality: Jurisdictional requirements which the mock_uss should follow. """ - if locality.is_uspace_applicable(): - # Validate flight authorisation - problems = problems_with_flight_authorisation(req_body.flight_authorisation) - if problems: - raise PlanningError(", ".join(problems)) - # Validate max number of vertices nb_vertices = 0 for volume in ( @@ -219,3 +215,23 @@ def op_intent_transition_valid( else: return False + + +def op_intent_from_flightrecord(flight: FlightRecord, method: str) -> OperationalIntent: + ref = flight.op_intent.reference + details = OperationalIntentDetails( + volumes=flight.op_intent.details.volumes, + off_nominal_volumes=flight.op_intent.details.off_nominal_volumes, + priority=flight.op_intent.details.priority, + ) + op_intent = OperationalIntent(reference=ref, details=details) + if flight.mod_op_sharing_behavior: + mod_op_sharing_behavior = flight.mod_op_sharing_behavior + if mod_op_sharing_behavior.modify_sharing_methods is not None: + if method not in mod_op_sharing_behavior.modify_sharing_methods: + return op_intent + op_intent = apply_overrides( + op_intent, mod_op_sharing_behavior.modify_fields, parse_result=False + ) + + return op_intent diff --git a/monitoring/mock_uss/scdsc/routes_scdsc.py b/monitoring/mock_uss/f3548v21/routes_scd.py similarity index 66% rename from monitoring/mock_uss/scdsc/routes_scdsc.py rename to monitoring/mock_uss/f3548v21/routes_scd.py index 4b2157a51e..0bb78bc04d 100644 --- a/monitoring/mock_uss/scdsc/routes_scdsc.py +++ b/monitoring/mock_uss/f3548v21/routes_scd.py @@ -1,19 +1,12 @@ -import json - import flask -from implicitdict import ImplicitDict + +from monitoring.mock_uss.f3548v21.flight_planning import op_intent_from_flightrecord from monitoring.monitorlib import scd from monitoring.mock_uss import webapp from monitoring.mock_uss.auth import requires_scope -from monitoring.mock_uss.scdsc.database import db -from monitoring.mock_uss.scdsc.database import FlightRecord -from monitoring.uss_qualifier.resources.overrides import ( - apply_overrides, -) +from monitoring.mock_uss.flights.database import db from uas_standards.astm.f3548.v21.api import ( ErrorResponse, - OperationalIntent, - OperationalIntentDetails, GetOperationalIntentDetailsResponse, ) @@ -51,26 +44,6 @@ def scdsc_get_operational_intent_details(entityid: str): return flask.jsonify(response), 200 -def op_intent_from_flightrecord(flight: FlightRecord, method: str) -> OperationalIntent: - ref = flight.op_intent.reference - details = OperationalIntentDetails( - volumes=flight.op_intent.details.volumes, - off_nominal_volumes=flight.op_intent.details.off_nominal_volumes, - priority=flight.op_intent.details.priority, - ) - op_intent = OperationalIntent(reference=ref, details=details) - if flight.mod_op_sharing_behavior: - mod_op_sharing_behavior = flight.mod_op_sharing_behavior - if mod_op_sharing_behavior.modify_sharing_methods is not None: - if method not in mod_op_sharing_behavior.modify_sharing_methods: - return op_intent - op_intent = apply_overrides( - op_intent, mod_op_sharing_behavior.modify_fields, parse_result=False - ) - - return op_intent - - @webapp.route("/mock/scd/uss/v1/operational_intents", methods=["POST"]) @requires_scope(scd.SCOPE_SC) def scdsc_notify_operational_intent_details_changed(): diff --git a/monitoring/mock_uss/flight_planning/README.md b/monitoring/mock_uss/flight_planning/README.md new file mode 100644 index 0000000000..c62c7a9b2b --- /dev/null +++ b/monitoring/mock_uss/flight_planning/README.md @@ -0,0 +1,3 @@ +# mock_uss: flight_planner + +This folder contains materials implementing [InterUSS's flight_planning automated testing interface](https://github.com/interuss/automated_testing_interfaces/tree/main/flight_planning) by [mock_uss](..). diff --git a/monitoring/mock_uss/flight_planning/routes.py b/monitoring/mock_uss/flight_planning/routes.py index 54f21caf15..0c7d90cf97 100644 --- a/monitoring/mock_uss/flight_planning/routes.py +++ b/monitoring/mock_uss/flight_planning/routes.py @@ -6,7 +6,7 @@ from implicitdict import ImplicitDict from loguru import logger -from monitoring.mock_uss.scdsc.routes_injection import ( +from monitoring.mock_uss.scd_injection.routes_injection import ( inject_flight, lock_flight, release_flight_lock, diff --git a/monitoring/mock_uss/flights/README.md b/monitoring/mock_uss/flights/README.md new file mode 100644 index 0000000000..05f35b8307 --- /dev/null +++ b/monitoring/mock_uss/flights/README.md @@ -0,0 +1,3 @@ +# mock_uss: flights + +This folder contains materials related to generic handling of user-requested flights by [mock_uss](..). diff --git a/monitoring/mock_uss/flights/__init__.py b/monitoring/mock_uss/flights/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/mock_uss/scdsc/database.py b/monitoring/mock_uss/flights/database.py similarity index 93% rename from monitoring/mock_uss/scdsc/database.py rename to monitoring/mock_uss/flights/database.py index 94892eadbe..aefeb2b9c7 100644 --- a/monitoring/mock_uss/scdsc/database.py +++ b/monitoring/mock_uss/flights/database.py @@ -1,4 +1,5 @@ import json +from datetime import timedelta from typing import Dict, Optional from monitoring.monitorlib.clients.flight_planning.flight_info import FlightInfo @@ -11,6 +12,8 @@ MockUssFlightBehavior, ) +DEADLOCK_TIMEOUT = timedelta(seconds=5) + class FlightRecord(ImplicitDict): """Representation of a flight in a USS""" diff --git a/monitoring/mock_uss/flights/planning.py b/monitoring/mock_uss/flights/planning.py new file mode 100644 index 0000000000..cc07cfd4bf --- /dev/null +++ b/monitoring/mock_uss/flights/planning.py @@ -0,0 +1,47 @@ +import time +from datetime import datetime +from typing import Callable + +from monitoring.mock_uss.flights.database import FlightRecord, db, DEADLOCK_TIMEOUT + + +def lock_flight(flight_id: str, log: Callable[[str], None]) -> FlightRecord: + # If this is a change to an existing flight, acquire lock to that flight + log(f"Acquiring lock for flight {flight_id}") + deadline = datetime.utcnow() + DEADLOCK_TIMEOUT + while True: + with db as tx: + if flight_id in tx.flights: + # This is an existing flight being modified + existing_flight = tx.flights[flight_id] + if existing_flight and not existing_flight.locked: + log("Existing flight locked for update") + existing_flight.locked = True + break + else: + log("Request is for a new flight (lock established)") + tx.flights[flight_id] = None + existing_flight = None + break + # We found an existing flight but it was locked; wait for it to become + # available + time.sleep(0.5) + + if datetime.utcnow() > deadline: + raise RuntimeError( + f"Deadlock in inject_flight while attempting to gain access to flight {flight_id}" + ) + return existing_flight + + +def release_flight_lock(flight_id: str, log: Callable[[str], None]) -> None: + with db as tx: + if flight_id in tx.flights: + if tx.flights[flight_id]: + # FlightRecord was a true existing flight + log(f"Releasing lock on existing flight_id {flight_id}") + tx.flights[flight_id].locked = False + else: + # FlightRecord was just a placeholder for a new flight + log(f"Releasing placeholder for existing flight_id {flight_id}") + del tx.flights[flight_id] diff --git a/monitoring/mock_uss/scd_injection/README.md b/monitoring/mock_uss/scd_injection/README.md new file mode 100644 index 0000000000..a6d634d284 --- /dev/null +++ b/monitoring/mock_uss/scd_injection/README.md @@ -0,0 +1,3 @@ +# mock_uss: scd_injection + +This folder contains material related to the deprecated [InterUSS scd automated testing interface](https://github.com/interuss/automated_testing_interfaces/tree/main/scd). diff --git a/monitoring/mock_uss/scd_injection/__init__.py b/monitoring/mock_uss/scd_injection/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/mock_uss/scd_injection/routes.py b/monitoring/mock_uss/scd_injection/routes.py new file mode 100644 index 0000000000..837bd9d2cb --- /dev/null +++ b/monitoring/mock_uss/scd_injection/routes.py @@ -0,0 +1,6 @@ +from monitoring.mock_uss import webapp + + +@webapp.route("/scdsc/status") +def scdsc_status(): + return "scd flight injection API ok" diff --git a/monitoring/mock_uss/scdsc/routes_injection.py b/monitoring/mock_uss/scd_injection/routes_injection.py similarity index 91% rename from monitoring/mock_uss/scdsc/routes_injection.py rename to monitoring/mock_uss/scd_injection/routes_injection.py index 1622fa76cc..db7e8d7afe 100644 --- a/monitoring/mock_uss/scdsc/routes_injection.py +++ b/monitoring/mock_uss/scd_injection/routes_injection.py @@ -2,7 +2,7 @@ import traceback from datetime import datetime, timedelta import time -from typing import List, Tuple, Callable, Optional +from typing import List, Tuple, Optional import uuid import flask @@ -10,6 +10,8 @@ from loguru import logger import requests.exceptions +from monitoring.mock_uss.flights.planning import lock_flight, release_flight_lock +from monitoring.mock_uss.f3548v21 import utm_client from monitoring.monitorlib.clients.flight_planning.flight_info import FlightInfo from uas_standards.astm.f3548.v21 import api from uas_standards.astm.f3548.v21.api import ( @@ -32,19 +34,20 @@ OperationalIntentState, ) -from monitoring.mock_uss import webapp, require_config_value +from monitoring.mock_uss import webapp, require_config_value, uspace from monitoring.mock_uss.auth import requires_scope from monitoring.mock_uss.config import KEY_BASE_URL from monitoring.mock_uss.dynamic_configuration.configuration import get_locality -from monitoring.mock_uss.scdsc import database, utm_client -from monitoring.mock_uss.scdsc.database import db, FlightRecord -from monitoring.mock_uss.scdsc.flight_planning import ( +from monitoring.mock_uss.flights import database +from monitoring.mock_uss.flights.database import db, FlightRecord +from monitoring.mock_uss.f3548v21.flight_planning import ( validate_request, check_for_disallowed_conflicts, PlanningError, + op_intent_from_flightrecord, op_intent_transition_valid, ) -from monitoring.mock_uss.scdsc.routes_scdsc import op_intent_from_flightrecord +import monitoring.mock_uss.uspace.flight_auth from monitoring.monitorlib import versioning from monitoring.monitorlib.clients import scd as scd_client from monitoring.monitorlib.fetch import QueryError @@ -188,48 +191,6 @@ def _mock_uss_flight_behavior_in_req( return None -def lock_flight(flight_id: str, log: Callable[[str], None]) -> FlightRecord: - # If this is a change to an existing flight, acquire lock to that flight - log(f"Acquiring lock for flight {flight_id}") - deadline = datetime.utcnow() + DEADLOCK_TIMEOUT - while True: - with db as tx: - if flight_id in tx.flights: - # This is an existing flight being modified - existing_flight = tx.flights[flight_id] - if existing_flight and not existing_flight.locked: - log("Existing flight locked for update") - existing_flight.locked = True - break - else: - log("Request is for a new flight (lock established)") - tx.flights[flight_id] = None - existing_flight = None - break - # We found an existing flight but it was locked; wait for it to become - # available - time.sleep(0.5) - - if datetime.utcnow() > deadline: - raise RuntimeError( - f"Deadlock in inject_flight while attempting to gain access to flight {flight_id}" - ) - return existing_flight - - -def release_flight_lock(flight_id: str, log: Callable[[str], None]) -> None: - with db as tx: - if flight_id in tx.flights: - if tx.flights[flight_id]: - # FlightRecord was a true existing flight - log(f"Releasing lock on existing flight_id {flight_id}") - tx.flights[flight_id].locked = False - else: - # FlightRecord was just a placeholder for a new flight - log(f"Releasing placeholder for existing flight_id {flight_id}") - del tx.flights[flight_id] - - def inject_flight( flight_id: str, req_body: MockUSSInjectFlightRequest, @@ -244,7 +205,9 @@ def log(msg: str): # Validate request log("Validating request") try: - validate_request(req_body, locality) + if locality.is_uspace_applicable(): + uspace.flight_auth.validate_request(req_body) + validate_request(req_body) except PlanningError as e: return ( InjectFlightResponse( diff --git a/monitoring/mock_uss/scdsc/README.md b/monitoring/mock_uss/scdsc/README.md deleted file mode 100644 index 1786fb2364..0000000000 --- a/monitoring/mock_uss/scdsc/README.md +++ /dev/null @@ -1 +0,0 @@ -[ASTM F3548-21](http://astm.org/f3548-21.html) standardizes UTM interoperability between USSs to achieve strategic coordination and communicate constraints. When this `scdsc` [mock_uss](..) functionality is enabled, mock_uss will behave like an F3548-21 strategic coordinator that accepts user flight planning attempts via the [InterUSS scd automated testing interface](../../../interfaces/automated_testing/scd) and attempts to establish and communicate an operational intent for those flights according to ASTM F3548-21. diff --git a/monitoring/mock_uss/scdsc/routes.py b/monitoring/mock_uss/scdsc/routes.py deleted file mode 100644 index 6c85debe12..0000000000 --- a/monitoring/mock_uss/scdsc/routes.py +++ /dev/null @@ -1,10 +0,0 @@ -from monitoring.mock_uss import webapp - - -@webapp.route("/scdsc/status") -def scdsc_status(): - return "Mock SCD strategic coordinator ok" - - -from . import routes_scdsc -from . import routes_injection diff --git a/monitoring/mock_uss/uspace/README.md b/monitoring/mock_uss/uspace/README.md new file mode 100644 index 0000000000..5bc1871b38 --- /dev/null +++ b/monitoring/mock_uss/uspace/README.md @@ -0,0 +1,3 @@ +# mock_uss: uspace + +This folder contains materials allowing mock_uss to emulate behaviors required in U-space. diff --git a/monitoring/mock_uss/uspace/__init__.py b/monitoring/mock_uss/uspace/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/mock_uss/uspace/flight_auth.py b/monitoring/mock_uss/uspace/flight_auth.py new file mode 100644 index 0000000000..5d6b3c989c --- /dev/null +++ b/monitoring/mock_uss/uspace/flight_auth.py @@ -0,0 +1,14 @@ +from monitoring.mock_uss.f3548v21.flight_planning import PlanningError +from monitoring.monitorlib.uspace import problems_with_flight_authorisation +from uas_standards.interuss.automated_testing.scd.v1 import api as scd_api + + +def validate_request(req_body: scd_api.InjectFlightRequest) -> None: + """Raise a PlannerError if the request is not valid. + + Args: + req_body: Information about the requested flight. + """ + problems = problems_with_flight_authorisation(req_body.flight_authorisation) + if problems: + raise PlanningError(", ".join(problems)) From 7f528d5f9f15076bf10adf258564ff0271a22613 Mon Sep 17 00:00:00 2001 From: Julien Perrochet Date: Tue, 31 Oct 2023 17:08:19 +0100 Subject: [PATCH 3/4] [uss_qualifier] DSS0030 port ISA-subscriptions interactions test (#291) * [uss-qualifier] DSS0030 port ISA-subscriptions interactions test * Add notification_index checks * rename CreateSubscription to PutSubscription for consistency * Reference the standard directly where relevant --- monitoring/monitorlib/fetch/rid.py | 11 + monitoring/prober/infrastructure.py | 2 +- .../dss/isa_subscription_interactions.py | 309 ++++++++++++++++++ .../common/dss/subscription_validation.py | 13 +- .../scenarios/astm/netrid/common/dss/utils.py | 31 +- .../scenarios/astm/netrid/v19/dss/__init__.py | 1 + .../v19/dss/isa_subscription_interactions.md | 148 +++++++++ .../v19/dss/isa_subscription_interactions.py | 8 + .../astm/netrid/v22a/dss/__init__.py | 1 + .../v22a/dss/isa_subscription_interactions.md | 148 +++++++++ .../v22a/dss/isa_subscription_interactions.py | 8 + .../suites/astm/netrid/f3411_19.md | 14 +- .../astm/netrid/f3411_19/dss_probing.md | 21 +- .../astm/netrid/f3411_19/dss_probing.yaml | 6 + .../suites/astm/netrid/f3411_22a.md | 14 +- .../astm/netrid/f3411_22a/dss_probing.md | 21 +- .../astm/netrid/f3411_22a/dss_probing.yaml | 6 + .../suites/interuss/dss/all_tests.md | 26 +- .../suites/uspace/network_identification.md | 14 +- .../suites/uspace/required_services.md | 14 +- 20 files changed, 742 insertions(+), 74 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.py diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index e6b26c046e..d8122b5404 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -603,6 +603,17 @@ def isa_url(self) -> str: f"Cannot retrieve isa_url using RID version {self.rid_version}" ) + @property + def notification_index(self) -> int: + if self.rid_version == RIDVersion.f3411_19: + return self.v19_value.notification_index + elif self.rid_version == RIDVersion.f3411_22a: + return self.v22a_value.notification_index + else: + raise NotImplementedError( + f"Cannot retrieve notification_index using RID version {self.rid_version}" + ) + class RIDQuery(ImplicitDict): v19_query: Optional[Query] = None diff --git a/monitoring/prober/infrastructure.py b/monitoring/prober/infrastructure.py index 9647a45a4c..bc73f0aba3 100644 --- a/monitoring/prober/infrastructure.py +++ b/monitoring/prober/infrastructure.py @@ -100,7 +100,7 @@ def wrapper_default_scope(*args, **kwargs): resource_type_code_descriptions: Dict[ResourceType, str] = {} -# Next code: 370 +# Next code: 371 def register_resource_type(code: int, description: str) -> ResourceType: """Register that the specified code refers to the described resource. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py new file mode 100644 index 0000000000..22ab4cf477 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py @@ -0,0 +1,309 @@ +from typing import Optional + +import arrow + +from monitoring.prober.infrastructure import register_resource_type +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource +from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource +from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss import utils +from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + + +class ISASubscriptionInteractions(GenericTestScenario): + """Based on the test_subscription_isa_interactions.py from the legacy prober tool.""" + + ISA_TYPE = register_resource_type(370, "ISA") + + def __init__( + self, + dss: DSSInstanceResource, + id_generator: IDGeneratorResource, + isa: ServiceAreaResource, + ): + super().__init__() + self._dss = ( + dss.dss_instance + ) # TODO: delete once _delete_isa_if_exists updated to use dss_wrapper + self._dss_wrapper = DSSWrapper(self, dss.dss_instance) + self._isa_id = id_generator.id_factory.make_id( + ISASubscriptionInteractions.ISA_TYPE + ) + # sub id is isa_id with last character replaced with '1' + # (the generated isa_id ends with a few '0's) + self._sub_id = self._isa_id[:-1] + "1" + self._isa_version: Optional[str] = None + self._isa = isa.specification + + now = arrow.utcnow().datetime + self._isa_start_time = self._isa.shifted_time_start(now) + self._isa_end_time = self._isa.shifted_time_end(now) + self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] + + def run(self): + self.begin_test_scenario() + + self._setup_case() + + self.begin_test_case("ISA Subscription Interactions") + self.begin_test_step("ISA Subscription Interactions") + + self._check_subscription_behaviors() + + self.end_test_step() + self.end_test_case() + self.end_test_scenario() + + def _check_subscription_behaviors(self): + """ + - Create an ISA. + - Create a subscription, response should include the pre-existing ISA and have a notification_index of 0. + - Modify the ISA, response should include the subscription with an incremented notification_index. + - Delete the ISA, response should include the subscription with an incremented notification_index. + - Delete the subscription. + """ + + # Create an ISA + with self.check("Create an ISA", [self._dss.participant_id]) as check: + created_isa = self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={200}, + area_vertices=self._isa_area, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=None, + ) + + # Create a subscription + with self.check( + "Create a subscription within the ISA footprint", [self._dss.participant_id] + ) as check: + created_subscription = self._dss_wrapper.put_sub( + check=check, + area_vertices=self._isa_area, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max, + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + sub_id=self._sub_id, + sub_version=None, + ) + + # Check the subscription + with self.check( + "Subscription for the ISA's area mentions the ISA", + [self._dss.participant_id], + ) as check: + if created_isa.dss_query.isa.id not in [ + isa.id for isa in created_subscription.isas + ]: + check.record_failed( + summary="Subscription response does not include the freshly created ISA", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"The subscription created for the area {self._isa_area} is expected to contain the ISA created for this same area. The returned subscription did not mention it.", + query_timestamps=[ + created_isa.dss_query.query.request.timestamp, + created_subscription.query.request.timestamp, + ], + ) + + with self.check( + "Newly created subscription has a notification_index of 0", + [self._dss.participant_id], + ) as check: + if created_subscription.subscription.notification_index != 0: + check.record_failed( + summary="Subscription notification_index is not 0", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 0. The returned subscription has a notification_index of {created_subscription.subscription.notification_index}.", + query_timestamps=[created_subscription.query.request.timestamp], + ) + + # Modify the ISA + with self.check( + "Mutate the ISA", + [self._dss.participant_id], + ) as check: + mutated_isa = self._dss_wrapper.put_isa_expect_response_code( + check=check, + expected_error_codes={200}, + area_vertices=self._isa_area, + alt_lo=self._isa.altitude_min, + alt_hi=self._isa.altitude_max - 1, # reduce max altitude by one meter + start_time=self._isa_start_time, + end_time=self._isa_end_time, + uss_base_url=self._isa.base_url, + isa_id=self._isa_id, + isa_version=created_isa.dss_query.isa.version, + ) + + # Check that the subscription ID is returned in the response + with self.check( + "Response to the mutation of the ISA contains subscription ID", + [self._dss.participant_id], + ) as check: + + subs_to_mutated_isa = {} + for returned_subscriber in mutated_isa.dss_query.subscribers: + for sub_in_subscriber in returned_subscriber.raw.subscriptions: + subs_to_mutated_isa[ + sub_in_subscriber.subscription_id + ] = sub_in_subscriber + + if created_subscription.subscription.id not in subs_to_mutated_isa.keys(): + check.record_failed( + summary="ISA mutation response does not contain expected subscription ID", + severity=Severity.High, + participants=[self._dss.participant_id], + details="Mutating an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.", + query_timestamps=[ + created_isa.dss_query.query.request.timestamp, + created_subscription.query.request.timestamp, + mutated_isa.dss_query.query.request.timestamp, + ], + ) + + # Check that the subscription index has been incremented by least by 1 + sub_to_mutated_isa = subs_to_mutated_isa.get( + created_subscription.subscription.id + ) + if sub_to_mutated_isa is not None: + with self.check( + "Subscription to an ISA has its notification index incremented after mutation", + [self._dss.participant_id], + ) as check: + if sub_to_mutated_isa.notification_index <= 0: + check.record_failed( + summary="Subscription notification_index has not been increased", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"The subscription created for the area {self._isa_area} is expected to have a notification_index of 1 or more. The returned subscription has a notification_index of {subs_to_mutated_isa[created_subscription.subscription.id].notification_index}.", + query_timestamps=[created_subscription.query.request.timestamp], + ) + + # Delete the ISA + with self.check( + "Delete the ISA", + [self._dss.participant_id], + ) as check: + deleted_isa = self._dss_wrapper.del_isa_expect_response_code( + main_check=check, + expected_error_codes={200}, + isa_id=mutated_isa.dss_query.isa.id, + isa_version=mutated_isa.dss_query.isa.version, + ) + + # Check response to deletion of ISA + with self.check( + "Response to the deletion of the ISA contains subscription ID", + [self._dss.participant_id], + ) as check: + + subs_to_deleted_isa = {} + for returned_subscriber in deleted_isa.dss_query.subscribers: + for sub_in_subscriber in returned_subscriber.raw.subscriptions: + subs_to_deleted_isa[ + sub_in_subscriber.subscription_id + ] = sub_in_subscriber + + if created_subscription.subscription.id not in subs_to_deleted_isa: + check.record_failed( + summary="ISA deletion response does not contain expected subscription ID", + severity=Severity.High, + participants=[self._dss.participant_id], + details="Deleting an ISA to which a subscription was made, the DSS failed to return the subscription ID in the response.", + query_timestamps=[ + created_isa.dss_query.query.request.timestamp, + created_subscription.query.request.timestamp, + deleted_isa.dss_query.query.request.timestamp, + ], + ) + + for subscriber_url, notification in deleted_isa.notifications.items(): + # For checking the notifications, we ignore the request we made for the subscription that we created. + if self._isa.base_url not in subscriber_url: + pid = notification.query.participant_id + with self.check("Notified subscriber", [pid] if pid else []) as check: + if not notification.success: + check.record_failed( + "Could not notify ISA subscriber", + Severity.Medium, + f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}", + query_timestamps=[notification.query.request.timestamp], + ) + + subs_after_deletion = subs_to_deleted_isa.get( + created_subscription.subscription.id + ) + if subs_after_deletion is not None: + with self.check( + "Subscription to an ISA has its notification index incremented after deletion", + [self._dss.participant_id], + ) as check: + if ( + subs_after_deletion.notification_index + <= sub_to_mutated_isa.notification_index + ): + check.record_failed( + summary="Subscription notification_index has not been incremented", + severity=Severity.High, + participants=[self._dss.participant_id], + details=f"The subscription created for the area {self._isa_area} is expected to have its notification increased after the subscription was deleted." + f"The returned subscription has a notification_index of {subs_after_deletion.notification_index}, whilte the previous notification_index for that subscription was {sub_to_mutated_isa.notification_index}", + query_timestamps=[created_subscription.query.request.timestamp], + ) + + # Delete the subscription + with self.check( + "Successful subscription deletion", + [self._dss.participant_id], + ) as check: + self._dss_wrapper.del_sub( + check=check, + sub_id=self._sub_id, + sub_version=created_subscription.subscription.version, + ) + + def _setup_case(self): + self.begin_test_case("Setup") + + def _ensure_clean_workspace_step(): + self.begin_test_step("Ensure clean workspace") + + self._delete_isa_if_exists() + self._clean_any_sub() + + self.end_test_step() + + _ensure_clean_workspace_step() + + self.end_test_case() + + def _delete_isa_if_exists(self): + utils.delete_isa_if_exists( + self, + isa_id=self._isa_id, + rid_version=self._dss.rid_version, + session=self._dss.client, + participant_id=self._dss_wrapper.participant_id, + ) + + def _clean_any_sub(self): + utils.delete_any_subscription(self, self._dss_wrapper, self._isa.footprint) + + def cleanup(self): + self.begin_cleanup() + + self._delete_isa_if_exists() + self._clean_any_sub() + + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py index fda32ab3e3..b2ef5419c3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py @@ -12,6 +12,7 @@ from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss import utils from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper from monitoring.uss_qualifier.scenarios.scenario import ( GenericTestScenario, @@ -77,17 +78,7 @@ def _setup_case(self): self.end_test_case() def _clean_any_sub(self): - with self.check( - "Successful subscription query", [self._dss.participant_id] - ) as check: - fetched = self._dss_wrapper.search_subs( - check, [vertex.as_s2sphere() for vertex in self._isa.footprint] - ) - for sub_id in fetched.subscriptions.keys(): - with self.check( - "Successful subscription deletion", [self._dss.participant_id] - ) as check: - self._dss_wrapper.cleanup_sub(check, sub_id=sub_id) + utils.delete_any_subscription(self, self._dss_wrapper, self._isa.footprint) def _ensure_clean_workspace_step(self): self.begin_test_step("Ensure clean workspace") diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py index ea314377e9..3a9627f86c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py @@ -1,10 +1,12 @@ -from typing import Optional +from typing import Optional, List from monitoring.monitorlib.fetch import rid as fetch +from monitoring.monitorlib.geo import LatLngPoint from monitoring.monitorlib.mutate import rid as mutate from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.rid import RIDVersion from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario @@ -50,6 +52,7 @@ def delete_isa_if_exists( f"Attempting to delete ISA {isa_id} from the {participant_id} DSS returned error {deleted.dss_query.status_code}", query_timestamps=[deleted.dss_query.query.request.timestamp], ) + for subscriber_url, notification in deleted.notifications.items(): with scenario.check("Notified subscriber", [subscriber_url]) as check: # TODO: Find a better way to identify a subscriber who couldn't be notified @@ -60,3 +63,29 @@ def delete_isa_if_exists( f"Attempting to notify subscriber for ISA {isa_id} at {subscriber_url} resulted in {notification.status_code}", query_timestamps=[notification.query.request.timestamp], ) + + +def delete_any_subscription( + scenario: GenericTestScenario, + dss_wrapper: DSSWrapper, + area: List[LatLngPoint], +): + """ + Deletes any subscription that is returned for the passed area. + + Args: + scenario: the scenario instance that will provide the checks + dss_wrapper: the dss on which to delete subscriptions + area: the area for which subscriptions are to be deleted + """ + with scenario.check( + "Successful subscription query", [dss_wrapper.participant_id] + ) as check: + fetched = dss_wrapper.search_subs( + check, [vertex.as_s2sphere() for vertex in area] + ) + for sub_id in fetched.subscriptions.keys(): + with scenario.check( + "Successful subscription deletion", [dss_wrapper.participant_id] + ) as check: + dss_wrapper.cleanup_sub(check, sub_id=sub_id) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py index 2b5fc635d5..77eade4f6e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py @@ -1,5 +1,6 @@ from .isa_simple import ISASimple from .isa_validation import ISAValidation from .isa_expiry import ISAExpiry +from .isa_subscription_interactions import ISASubscriptionInteractions from .subscription_validation import SubscriptionValidation from .crdb_access import CRDBAccess diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md new file mode 100644 index 0000000000..db583beff8 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md @@ -0,0 +1,148 @@ +# ASTM NetRID DSS: ISA Subscription Interactions test scenario + +## Overview + +Verifies that interactions between ISAs and subscriptions happen as expected. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing an ISA to be created. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates an ISA with a known ID. This step ensures that the ISA does not exist when the main part of the test starts. + +Any previously created subscriptions for the test ISA's area that might still exist will be deleted. + +#### Successful ISA query check + +**[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. + +#### Successful subscription query check + +**[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. + +#### Successful subscription deletion check + +**[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. + +## ISA Subscription Interactions test case + +This test case will do the following, using the DSS being tested: + +1. Create an ISA with the configured footprint, +2. Create a subscription for the ISA's area, and expect: + - to find the created ISA mentioned in the reply + - the notification index of the subscription to be 0 +3. Modify the ISA, and expect: + - to find the created subscription in the reply + - the notification index of the subscription to be greater than 0 +4. Delete the ISA, and expect: + - to find the created subscription in the reply + - the notification index of the subscription to be greater than it was after the mutation +5. Delete the subscription. + +### ISA Subscription Interactions test step + +#### Create an ISA check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### Create a subscription within the ISA footprint check + +The DSS should allow the creation of a subscription within the ISA footprint, otherwise it is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** + +#### Subscription for the ISA's area mentions the ISA check + +A subscription that is created for a volume that intersects with the previously created ISA should mention +the previously created ISA. If not, the serving DSS is in violation of **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. + +#### Newly created subscription has a notification_index of 0 check + +A newly created subscription is expected to have a notification index of 0, otherwise the DSS implementation under +test does not comply with **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** + +#### Mutate the ISA check + +If the ISA cannot be mutated, **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. + +#### Response to the mutation of the ISA contains subscription ID check + +When an ISA is mutated, the DSS must return the identifiers for any subscription that was made to the ISA, +or be in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription to an ISA has its notification index incremented after mutation check + +When an ISA is mutated, the DSS must increment the notification index of any subscription to that ISA, +and return the up-to-date subscription in the response to the query mutating the ISA. + +Failure to do so means that the DSS is not properly implementing **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Delete the ISA check + +If that ISA cannot be deleted, the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Response to the deletion of the ISA contains subscription ID check + +When an ISA is deleted, the DSS must return the identifiers for any subscription that was made to the ISA, +or be in violation of **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**. + +#### Subscription to an ISA has its notification index incremented after deletion check + +When an ISA is deleted, the DSS must increment the notification index of any subscription to that ISA, +and return the up-to-date subscription in the response to the query deleting the ISA. + +Failure to do so means that the DSS is not properly implementing **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. + +#### Successful subscription deletion check + +**[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. + +#### Notified subscriber check + +Notifications to any subscriber to the created ISA need to be successful. If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely while +also deleting any subscription it might have created for the ISA's area. + +#### Successful ISA query check + +**[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. + +#### Successful subscription query check + +**[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. + +#### Successful subscription deletion check + +**[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py new file mode 100644 index 0000000000..b6acd67db8 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.isa_subscription_interactions import ( + ISASubscriptionInteractions as CommonISASubscriptionInteractions, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class ISASubscriptionInteractions(TestScenario, CommonISASubscriptionInteractions): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py index 2b5fc635d5..77eade4f6e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py @@ -1,5 +1,6 @@ from .isa_simple import ISASimple from .isa_validation import ISAValidation from .isa_expiry import ISAExpiry +from .isa_subscription_interactions import ISASubscriptionInteractions from .subscription_validation import SubscriptionValidation from .crdb_access import CRDBAccess diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md new file mode 100644 index 0000000000..f4e9e86e40 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md @@ -0,0 +1,148 @@ +# ASTM NetRID DSS: ISA Subscription Interactions test scenario + +## Overview + +Verifies that interactions between ISAs and subscriptions happen as expected. + +## Resources + +### dss + +[`DSSInstanceResource`](../../../../../resources/astm/f3411/dss.py) to be tested in this scenario. + +### id_generator + +[`IDGeneratorResource`](../../../../../resources/interuss/id_generator.py) providing the ISA ID for this scenario. + +### isa + +[`ServiceAreaResource`](../../../../../resources/netrid/service_area.py) describing an ISA to be created. + +## Setup test case + +### Ensure clean workspace test step + +This scenario creates an ISA with a known ID. This step ensures that the ISA does not exist when the main part of the test starts. + +Any previously created subscriptions for the test ISA's area that might still exist will be deleted. + +#### Successful ISA query check + +**[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. + +#### Successful subscription query check + +**[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. + +#### Successful subscription deletion check + +**[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. + +## ISA Subscription Interactions test case + +This test case will do the following, using the DSS being tested: + +1. Create an ISA with the configured footprint, +2. Create a subscription for the ISA's area, and expect: + - to find the created ISA mentioned in the reply + - the notification index of the subscription to be 0 +3. Modify the ISA, and expect: + - to find the created subscription in the reply + - the notification index of the subscription to be greater than 0 +4. Delete the ISA, and expect: + - to find the created subscription in the reply + - the notification index of the subscription to be greater than it was after the mutation +5. Delete the subscription. + +### ISA Subscription Interactions test step + +#### Create an ISA check + +If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### Create a subscription within the ISA footprint check + +The DSS should allow the creation of a subscription within the ISA footprint, otherwise it is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** + +#### Subscription for the ISA's area mentions the ISA check + +A subscription that is created for a volume that intersects with the previously created ISA should mention +the previously created ISA. If not, the serving DSS is in violation of **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Newly created subscription has a notification_index of 0 check + +A newly created subscription is expected to have a notification index of 0, otherwise the DSS implementation under +test does not comply with **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** + +#### Mutate the ISA check + +If the ISA cannot be mutated, **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. + +#### Response to the mutation of the ISA contains subscription ID check + +When an ISA is mutated, the DSS must return the identifiers for any subscription that was made to the ISA, +or be in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription to an ISA has its notification index incremented after mutation check + +When an ISA is mutated, the DSS must increment the notification index of any subscription to that ISA, +and return the up-to-date subscription in the response to the query mutating the ISA. + +Failure to do so means that the DSS is not properly implementing **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Delete the ISA check + +If that ISA cannot be deleted, the **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Response to the deletion of the ISA contains subscription ID check + +When an ISA is deleted, the DSS must return the identifiers for any subscription that was made to the ISA, +or be in violation of **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Subscription to an ISA has its notification index incremented after deletion check + +When an ISA is deleted, the DSS must increment the notification index of any subscription to that ISA, +and return the up-to-date subscription in the response to the query deleting the ISA. + +Failure to do so means that the DSS is not properly implementing **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. + +#### Successful subscription deletion check + +**[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. + +#### Notified subscriber check + +Notifications to any subscriber to the created ISA need to be successful. If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. + +## Cleanup + +The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely while +also deleting any subscription it might have created for the ISA's area. + +#### Successful ISA query check + +**[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. + +#### Removed pre-existing ISA check + +If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. + +#### Notified subscriber check + +When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. + +#### Successful subscription query check + +**[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. + +#### Successful subscription deletion check + +**[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.py new file mode 100644 index 0000000000..b6acd67db8 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.isa_subscription_interactions import ( + ISASubscriptionInteractions as CommonISASubscriptionInteractions, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class ISASubscriptionInteractions(TestScenario, CommonISASubscriptionInteractions): + pass diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 5866e3105c..0b46d1d692 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -24,27 +24,27 @@ astm
.f3411
.v19 DSS0030,a Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -324,7 +324,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.automated_testing
.rid
.injection @@ -362,7 +362,7 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index d834ea7d80..af009a680c 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -7,9 +7,10 @@ 1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v19/dss/isa_simple.md) ([`scenarios.astm.netrid.v19.dss.ISASimple`](../../../../scenarios/astm/netrid/v19/dss/isa_simple.py)) 2. Scenario: [ASTM NetRID DSS: Submitted ISA Validations](../../../../scenarios/astm/netrid/v19/dss/isa_validation.md) ([`scenarios.astm.netrid.v19.dss.ISAValidation`](../../../../scenarios/astm/netrid/v19/dss/isa_validation.py)) 3. Scenario: [ASTM NetRID DSS: ISA Expiry](../../../../scenarios/astm/netrid/v19/dss/isa_expiry.md) ([`scenarios.astm.netrid.v19.dss.ISAExpiry`](../../../../scenarios/astm/netrid/v19/dss/isa_expiry.py)) -4. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) -5. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../../scenarios/astm/netrid/v19/dss_interoperability.py)) -6. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) +4. Scenario: [ASTM NetRID DSS: ISA Subscription Interactions](../../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md) ([`scenarios.astm.netrid.v19.dss.ISASubscriptionInteractions`](../../../../scenarios/astm/netrid/v19/dss/isa_subscription_interactions.py)) +5. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) +6. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../../scenarios/astm/netrid/v19/dss_interoperability.py)) +7. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) ## [Checked requirements](../../../README.md#checked-requirements) @@ -24,27 +25,27 @@ astm
.f3411
.v19 DSS0030,a Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -199,13 +200,13 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml index d8a6128878..26b8967611 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.yaml @@ -26,6 +26,12 @@ actions: dss: dss id_generator: id_generator isa: isa + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.dss.ISASubscriptionInteractions + resources: + dss: dss + id_generator: id_generator + isa: isa - test_scenario: scenario_type: scenarios.astm.netrid.v19.dss.SubscriptionValidation resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 4180056274..574c121027 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -29,27 +29,27 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -449,7 +449,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.automated_testing
.rid
.injection @@ -487,7 +487,7 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md index 375ff330c9..35c5476bfd 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md @@ -7,9 +7,10 @@ 1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) 2. Scenario: [ASTM NetRID DSS: Submitted ISA Validations](../../../../scenarios/astm/netrid/v22a/dss/isa_validation.md) ([`scenarios.astm.netrid.v22a.dss.ISAValidation`](../../../../scenarios/astm/netrid/v22a/dss/isa_validation.py)) 3. Scenario: [ASTM NetRID DSS: ISA Expiry](../../../../scenarios/astm/netrid/v22a/dss/isa_expiry.md) ([`scenarios.astm.netrid.v22a.dss.ISAExpiry`](../../../../scenarios/astm/netrid/v22a/dss/isa_expiry.py)) -4. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) -5. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) -6. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v22a/dss/crdb_access.md) ([`scenarios.astm.netrid.v22a.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v22a/dss/crdb_access.py)) +4. Scenario: [ASTM NetRID DSS: ISA Subscription Interactions](../../../../scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md) ([`scenarios.astm.netrid.v22a.dss.ISASubscriptionInteractions`](../../../../scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.py)) +5. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) +6. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) +7. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../../scenarios/astm/netrid/v22a/dss/crdb_access.md) ([`scenarios.astm.netrid.v22a.dss.CRDBAccess`](../../../../scenarios/astm/netrid/v22a/dss/crdb_access.py)) ## [Checked requirements](../../../README.md#checked-requirements) @@ -29,27 +30,27 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -209,13 +210,13 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml index d71c1f72dc..6b159abb5e 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.yaml @@ -26,6 +26,12 @@ actions: dss: dss id_generator: id_generator isa: isa + - test_scenario: + scenario_type: scenarios.astm.netrid.v22a.dss.ISASubscriptionInteractions + resources: + dss: dss + id_generator: id_generator + isa: isa - test_scenario: scenario_type: scenarios.astm.netrid.v22a.dss.SubscriptionValidation resources: diff --git a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md index 9f2ca89d2f..f582f9ab1b 100644 --- a/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md +++ b/monitoring/uss_qualifier/suites/interuss/dss/all_tests.md @@ -22,27 +22,27 @@ astm
.f3411
.v19 DSS0030,a Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-19 NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -197,7 +197,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations astm
.f3411
.v22a @@ -208,27 +208,27 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -388,13 +388,13 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index f99a13245d..28230fd3ac 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -24,27 +24,27 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -444,7 +444,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations interuss
.automated_testing
.rid
.injection @@ -482,7 +482,7 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index d72dfe8232..33c2a97919 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -26,27 +26,27 @@ DSS0030,a Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA DSS0030,b Implemented - ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM F3411-22a NetRID DSS interoperability
ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations DSS0030,c Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,d Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0030,f Implemented - ASTM NetRID DSS: Subscription Validation + ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Subscription Validation DSS0050 @@ -446,7 +446,7 @@ NET0730 Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations astm
.f3548
.v21 @@ -606,7 +606,7 @@ interuss
.f3411
.dss_endpoints GetISA Implemented - ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations + ASTM NetRID DSS: ISA Expiry
ASTM NetRID DSS: ISA Subscription Interactions
ASTM NetRID DSS: Simple ISA
ASTM NetRID DSS: Submitted ISA Validations SearchISAs From ea07fd956d044c838cc072b6a77cd10636a6a63c Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Tue, 31 Oct 2023 09:46:27 -0700 Subject: [PATCH 4/4] [uss_qualifier] Call flight planner from general flight authorization scenario (#297) * Call flight planner from general flight authorization scenario * Fix import location --- github_pages/static/index.md | 5 + .../clients/flight_planning/client_v1.py | 39 +++- monitoring/monitorlib/fetch/__init__.py | 14 ++ .../dev/general_flight_auth.yaml | 21 ++ .../configurations/dev/library/resources.yaml | 158 ++++++++------ monitoring/uss_qualifier/reports/report.py | 6 +- .../automated_testing/flight_planning.md | 13 +- .../flight_planning/flight_planner.py | 34 +-- .../flight_planning/flight_planners.py | 5 + .../flight_authorization/definitions.py | 18 +- .../general_flight_authorization.md | 12 ++ .../general_flight_authorization.py | 200 +++++++++++++++--- .../ASTMF354821OpIntentInformation.json | 18 ++ .../flight_info/FlightAuthorisationData.json | 98 +++++++++ .../flight_info/RPAS26FlightDetails.json | 102 +++++++++ .../BasicFlightPlanInformationTemplate.json | 41 ++++ .../FlightInfoTemplate.json | 55 +++++ .../monitoring/monitorlib/fetch/Query.json | 6 +- .../definitions/FlightCheck.json | 20 +- 19 files changed, 730 insertions(+), 135 deletions(-) create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json diff --git a/github_pages/static/index.md b/github_pages/static/index.md index 02cd5b3ce0..9b9bf2290b 100644 --- a/github_pages/static/index.md +++ b/github_pages/static/index.md @@ -33,3 +33,8 @@ These reports were generated during continuous integration for the most recent P * [Sequence view](./artifacts/uss_qualifier/reports/dss_probing/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/dss_probing/requirements) + +### [General flight authorization configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml) + +* [Sequence view](./artifacts/uss_qualifier/reports/general_flight_auth/sequence) +* [Tested requirements](./artifacts/uss_qualifier/reports/general_flight_auth/requirements) diff --git a/monitoring/monitorlib/clients/flight_planning/client_v1.py b/monitoring/monitorlib/clients/flight_planning/client_v1.py index 8fa3cf32bb..ce91c33182 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_v1.py +++ b/monitoring/monitorlib/clients/flight_planning/client_v1.py @@ -19,9 +19,10 @@ from monitoring.monitorlib.clients.flight_planning.planning import ( PlanningActivityResponse, ) -from monitoring.monitorlib.fetch import query_and_describe +from monitoring.monitorlib.fetch import query_and_describe, QueryType from monitoring.monitorlib.geotemporal import Volume4D from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.uss_qualifier.configurations.configuration import ParticipantID from uas_standards.interuss.automated_testing.flight_planning.v1 import api from uas_standards.interuss.automated_testing.flight_planning.v1.constants import Scope @@ -29,9 +30,11 @@ class V1FlightPlannerClient(FlightPlannerClient): _session: UTMClientSession + _participant_id: ParticipantID - def __init__(self, session: UTMClientSession): + def __init__(self, session: UTMClientSession, participant_id: ParticipantID): self._session = session + self._participant_id = participant_id def _inject( self, @@ -53,7 +56,13 @@ def _inject( op = api.OPERATIONS[api.OperationID.UpsertFlightPlan] url = op.path.format(flight_plan_id=flight_plan_id) query = query_and_describe( - self._session, op.verb, url, json=req, scope=Scope.Plan + self._session, + op.verb, + url, + json=req, + scope=Scope.Plan, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1UpsertFlightPlan, ) if query.status_code != 200 and query.status_code != 201: raise PlanningActivityError( @@ -108,7 +117,14 @@ def try_end_flight( ) op = api.OPERATIONS[api.OperationID.DeleteFlightPlan] url = op.path.format(flight_plan_id=flight_id) - query = query_and_describe(self._session, op.verb, url, scope=Scope.Plan) + query = query_and_describe( + self._session, + op.verb, + url, + scope=Scope.Plan, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1DeleteFlightPlan, + ) if query.status_code != 200: raise PlanningActivityError( f"Attempt to delete flight plan returned status {query.status_code} rather than 200 as expected", @@ -134,7 +150,12 @@ def try_end_flight( def report_readiness(self) -> TestPreparationActivityResponse: op = api.OPERATIONS[api.OperationID.GetStatus] query = query_and_describe( - self._session, op.verb, op.path, scope=Scope.DirectAutomatedTest + self._session, + op.verb, + op.path, + scope=Scope.DirectAutomatedTest, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1GetStatus, ) if query.status_code != 200: raise PlanningActivityError( @@ -166,7 +187,13 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: op = api.OPERATIONS[api.OperationID.ClearArea] query = query_and_describe( - self._session, op.verb, op.path, json=req, scope=Scope.DirectAutomatedTest + self._session, + op.verb, + op.path, + json=req, + scope=Scope.DirectAutomatedTest, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1ClearArea, ) if query.status_code != 200: raise PlanningActivityError( diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 1661f0eb33..d69a3eb321 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -221,6 +221,20 @@ class QueryType(str, Enum): # InterUSS automated testing versioning interface InterUSSVersioningGetVersion = "interuss.automated_testing.versioning.GetVersion" + # InterUSS automated testing flight_planning interface + InterUSSFlightPlanningV1GetStatus = ( + "interuss.automated_testing.flight_planning.v1.GetStatus" + ) + InterUSSFlightPlanningV1ClearArea = ( + "interuss.automated_testing.flight_planning.v1.ClearArea" + ) + InterUSSFlightPlanningV1UpsertFlightPlan = ( + "interuss.automated_testing.flight_planning.v1.UpsertFlightPlan" + ) + InterUSSFlightPlanningV1DeleteFlightPlan = ( + "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" + ) + @staticmethod def flight_details(rid_version: RIDVersion): if rid_version == RIDVersion.f3411_19: diff --git a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml index eda18dbd56..e1d39e5280 100644 --- a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml +++ b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml @@ -4,13 +4,34 @@ v1: resources: resource_declarations: example_flight_check_table: {$ref: 'library/resources.yaml#/example_flight_check_table'} + + utm_auth: {$ref: 'library/environment.yaml#/utm_auth'} + uss1_flight_planner: {$ref: 'library/environment.yaml#/uss1_flight_planner'} + non_baseline_inputs: + - v1.test_run.resources.resource_declarations.utm_auth + - v1.test_run.resources.resource_declarations.uss1_flight_planner action: test_scenario: scenario_type: scenarios.interuss.flight_authorization.GeneralFlightAuthorization resources: table: example_flight_check_table + planner: uss1_flight_planner artifacts: output_path: output/general_flight_auth raw_report: {} + sequence_view: {} + tested_requirements: + - report_name: requirements + requirement_collections: + example: + requirement_collections: + - requirements: + - REQ_001 + - REQ_002 + - REQ_003 + - REQ_004 + - REQ_007 + participant_requirements: + uss1: example validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index b3667c5a28..0fee0615c5 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -151,73 +151,113 @@ example_flight_check_table: - REQ_002 - REQ_007 description: The first test step defined by the test designer - additional_information: - new_jurisdiction_x: - operation_rule_set: Rules1 - volumes: - - outline_circle: - center: - lng: 7.4774 - lat: 46.9749 - radius: - value: 100 - units: M - altitude_lower: - value: 0 - units: M - reference: SFC - altitude_upper: - value: 100 - units: M - reference: SFC - start_time: - start_of_test: { } - use_timezone: Europe/Berlin - end_time: - offset_from: - starting_from: - next_day: - time_zone: Europe/Zurich - starting_from: - start_of_test: { } - days_of_the_week: [ "Tu", "Th" ] - offset: 12h acceptance_expectation: MustBeAccepted + flight_info: + basic_information: + usage_state: Planned + uas_state: Nominal + area: + - outline_circle: + center: + lng: 7.4774 + lat: 46.9749 + radius: + value: 100 + units: M + altitude_lower: + value: 550 + units: M + # TODO: Change to SFC once mock_uss can process that datum + reference: W84 + altitude_upper: + value: 650 + units: M + # TODO: Change to SFC once mock_uss can process that datum + reference: W84 + start_time: + start_of_test: { } + use_timezone: Europe/Berlin + end_time: + offset_from: + starting_from: + next_day: + time_zone: Europe/Zurich + starting_from: + start_of_test: { } + days_of_the_week: [ "Tu", "Th" ] + offset: 12h + additional_information: + new_jurisdiction_x: + operation_rule_set: Rules1 + # TODO: Remove once mock_uss is fixed to not require U-space flight auth + uspace_flight_authorisation: + uas_serial_number: 1AF49UL5CC5J6K + operation_category: Open + operation_mode: Vlos + uas_class: C0 + identification_technologies: + - ASTMNetRID + connectivity_methods: + - cellular + endurance_minutes: 30 + emergency_procedure_url: https://example.interussplatform.org/emergency + operator_id: CHEo5kut30e0mt01-qwe + uas_id: '' + uas_type_certificate: '' - flight_check_id: TEST_002 requirement_ids: - REQ_001 - REQ_003 - REQ_004 description: The second test step defined by the test designer - additional_information: - new_jurisdiction_x: - operation_rule_set: Rules1 - volumes: - - outline_circle: - center: - lng: 7.4774 - lat: 46.9749 - radius: - value: 100 - units: M - altitude_lower: - value: 50 - units: M - reference: SFC - altitude_upper: - value: 5000 - units: FT - reference: W84 - start_time: - next_day: - time_zone: +02:00 - starting_from: - offset_from: + acceptance_expectation: MustBeAccepted + flight_info: + basic_information: + usage_state: Planned + uas_state: Nominal + area: + - outline_circle: + center: + lng: 7.4774 + lat: 46.9749 + radius: + value: 100 + units: M + altitude_lower: + value: 1424 + units: M + reference: W84 + altitude_upper: + value: 5000 + units: FT + reference: W84 + start_time: + next_day: + time_zone: +02:00 starting_from: - start_of_test: { } - offset: 12h - duration: 5m - conditions_expectation: MustBePresent + offset_from: + starting_from: + start_of_test: { } + offset: 12h + duration: 5m + additional_information: + new_jurisdiction_x: + operation_rule_set: Rules1 + # TODO: Remove once mock_uss is fixed to not require U-space flight auth + uspace_flight_authorisation: + uas_serial_number: 1AF49UL5CC5J6K + operation_category: Open + operation_mode: Vlos + uas_class: C0 + identification_technologies: + - ASTMNetRID + connectivity_methods: + - cellular + endurance_minutes: 30 + emergency_procedure_url: https://example.interussplatform.org/emergency + operator_id: CHEo5kut30e0mt01-qwe + uas_id: '' + uas_type_certificate: '' # ===== Geospatial feature comprehension ===== diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index fa9a110d98..91775d7213 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -457,7 +457,7 @@ def query_passed_checks( def query_failed_checks( self, participant_id: Optional[str] = None - ) -> Iterator[Tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[Tuple[JSONPathExpression, FailedCheck]]: test_suite, test_scenario, action_generator = self.get_applicable_report() if test_suite: report = self.test_suite @@ -486,7 +486,9 @@ def start_time(self) -> Optional[StringBasedDateTime]: @property def end_time(self) -> Optional[StringBasedDateTime]: - return self._conditional(lambda report: report.end_time) + return self._conditional( + lambda report: report.end_time if "end_time" in report else None + ) class AllConditionsEvaluationReport(ImplicitDict): diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md index 63c7169b31..b401fdad94 100644 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md @@ -2,19 +2,28 @@ ## Overview -TODO: Link to API YAML and provide overview +When a USS implements the [InterUSS flight_planning automated testing API](https://github.com/interuss/automated_testing_interfaces/tree/main/flight_planning) (or [legacy scd automated testing API](https://github.com/interuss/automated_testing_interfaces/tree/main/scd)), they are expected to respond to requests to that API as defined in the API. Specific requirements are below. ## Requirements -TODO: Describe requirements +### ImplementAPI + +A USS must implement the endpoints defined in the API, accept requests in the data format prescribed in the API, and respond in the data format prescribed in the API. If there is a problem using the API such as a connection error, invalid response code, or invalid data, the USS will have failed to meet this requirement. ### ClearArea +In order to conduct automated tests effectively, the USS must remove all of their existing flights from a particular area when instructed by the test director. This is not an action performed on behalf of an emulated user, but rather an action performed in any way appropriate to support automated testing -- therefore, fulfilling this request may cause actions on the implementing USS's system that no normal user would be able to perform. + ### ExpectedBehavior +When the test director (client of the flight planning API; usually uss_qualifier) requests that a flight planning activity be performed, the API implementer must act as if this request is coming from a normal user attempting to use the USS's system normally. The USS must fulfill this request as it would for a normal user, and these actions are generally expected to succeed (allowing the user to fly) when a UTM rule does not prohibit them. + ### FlightCoveredByOperationalIntent + For InterUSS to effectively test the requirements of ASTM F3548-21, a USS under test must act as if there is a regulatory requirement requiring all flights it manages to provide operational intents according to ASTM F3548-21 at all times for all flights it manages. ### DeleteFlightSuccess + +In order to conduct automated tests effectively, the USS must remove a particular flight when instructed by the test director. This is not an action performed on behalf of an emulated user, but rather an action performed in any way appropriate to support automated testing -- therefore, fulfilling this request may cause actions on the implementing USS's system that no normal user would be able to perform. diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py index 70d1aa2cb7..3ca0a44645 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py @@ -3,7 +3,10 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import infrastructure, fetch -from monitoring.monitorlib.clients.flight_planning.client import PlanningActivityError +from monitoring.monitorlib.clients.flight_planning.client import ( + PlanningActivityError, + FlightPlannerClient, +) from monitoring.monitorlib.clients.flight_planning.client_scd import ( SCDFlightPlannerClient, ) @@ -32,7 +35,6 @@ DeleteFlightResponse, InjectFlightRequest, ClearAreaResponse, - ClearAreaRequest, OperationalIntentState, ClearAreaOutcome, ) @@ -70,6 +72,23 @@ def __init__(self, *args, **kwargs): except ValueError: raise ValueError("FlightPlannerConfiguration.v1_base_url must be a URL") + def to_client( + self, auth_adapter: infrastructure.AuthAdapter + ) -> FlightPlannerClient: + if "scd_injection_base_url" in self and self.scd_injection_base_url: + session = infrastructure.UTMClientSession( + self.scd_injection_base_url, auth_adapter, self.timeout_seconds + ) + return SCDFlightPlannerClient(session) + elif "v1_base_url" in self and self.v1_base_url: + session = infrastructure.UTMClientSession( + self.v1_base_url, auth_adapter, self.timeout_seconds + ) + return V1FlightPlannerClient(session, self.participant_id) + raise ValueError( + "Could not construct FlightPlannerClient from provided configuration" + ) + class FlightPlanner: """Manages the state and the interactions with flight planner USS. @@ -82,16 +101,7 @@ def __init__( auth_adapter: infrastructure.AuthAdapter, ): self.config = config - if "scd_injection_base_url" in config and config.scd_injection_base_url: - session = infrastructure.UTMClientSession( - self.config.scd_injection_base_url, auth_adapter, config.timeout_seconds - ) - self.client = SCDFlightPlannerClient(session) - elif "v1_base_url" in config and config.v1_base_url: - session = infrastructure.UTMClientSession( - self.config.v1_base_url, auth_adapter, config.timeout_seconds - ) - self.client = V1FlightPlannerClient(session) + self.client = config.to_client(auth_adapter) # Flights injected by this target. self.created_flight_ids: Set[str] = set() diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py index 92595397e9..d61ad2351e 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py @@ -1,6 +1,7 @@ from typing import List, Iterable, Dict, Optional from implicitdict import ImplicitDict +from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient from monitoring.uss_qualifier.reports.report import ParticipantID from monitoring.uss_qualifier.resources.definitions import ResourceID @@ -18,6 +19,8 @@ class FlightPlannerSpecification(ImplicitDict): class FlightPlannerResource(Resource[FlightPlannerSpecification]): flight_planner: FlightPlanner + client: FlightPlannerClient + participant_id: ParticipantID def __init__( self, @@ -27,6 +30,8 @@ def __init__( self.flight_planner = FlightPlanner( specification.flight_planner, auth_adapter.adapter ) + self.client = specification.flight_planner.to_client(auth_adapter.adapter) + self.participant_id = specification.flight_planner.participant_id class FlightPlannersSpecification(ImplicitDict): diff --git a/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py b/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py index ee9b7827ae..d73ca3e601 100644 --- a/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py +++ b/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py @@ -1,8 +1,10 @@ from enum import Enum -from typing import List, Optional +from typing import List from implicitdict import ImplicitDict -from monitoring.monitorlib.geotemporal import Volume4DTemplate +from monitoring.monitorlib.clients.flight_planning.flight_info_template import ( + FlightInfoTemplate, +) class AcceptanceExpectation(str, Enum): @@ -37,16 +39,8 @@ class FlightCheck(ImplicitDict): description: str """Human-readable test step description to aid in the debugging and traceability.""" - volumes: List[Volume4DTemplate] - """Spatial and temporal definition of the areas the virtual user intends to fly in. - - A service provider is expected to authorizing a flight covering the entire area specified and for any of the entire time specified. - """ - - additional_information: dict - """Any additional information that should be provided to a USS planning the flight. - - Format is agreed upon between test designer and USSs.""" + flight_info: FlightInfoTemplate + """Information about the flight, as a user would provide it to the USS.""" acceptance_expectation: AcceptanceExpectation = AcceptanceExpectation.Irrelevant """Expected outcome when authorizing a flight as described.""" diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md index 4560f36389..2f282ab3e0 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md @@ -10,12 +10,20 @@ This test acts as a user using a USS's flight planning/authorization interface a [Flight Check Table](../../../resources/interuss/flight_authorization/flight_check_table.py) consisting of a list of Flight Check rows. Each Flight Check row will cause this test to attempt to plan/authorize a flight using the planning/authorization interfaces of each USS under test according to the information in that Flight Check row. This test will then perform checks according to the expected outcomes from those planning/authorization attempts, according to the Flight Check row. +### planner + +[Flight planner](../../../resources/flight_planning/flight_planners.py) providing access to the flight-planning USS under test in this scenario. + ## Flight planning test case ### Dynamic test step The test steps for this test scenario are generated dynamically according to the definitions in the Flight Check Table. The checks for each step are the same and are documented below. +#### Valid planning response check + +The USS under test is expected to implement the InterUSS flight_planning automated testing API and respond to requests accordingly. If the USS does not respond to a flight planning request to this API properly, it will have failed to meet **[interuss.automated_testing.flight_planning.ImplementAPI](../../../requirements/interuss/automated_testing/flight_planning.md)**. + #### Disallowed flight check When the test designer specifies that a particular Flight Check has an expected acceptance of "No", that means attempting to plan/authorize that flight in a USS should result in the request being rejected. Upon this test making this request, if the USS successfully plans/authorizes the flight, this check will fail. @@ -31,3 +39,7 @@ When the test designer specifies that a particular Flight Check's conditions "Mu #### Disallowed conditions check When the test designer specifies that a particular Flight Check's conditions "MustBeAbsent", that means if a flight is successfully planned/authorized, it must NOT be accompanied by any conditions/advisories. If a successfully-planned/authorized flight IS indicated to contain any conditions/advisories, this check will fail. + +#### Successful closure check + +If a flight was successfully planned, then uss_qualifier will emulate a user attempting to close that flight. The flight plan is expected to be Closed following that action. If it is any other value, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../requirements/interuss/automated_testing/flight_planning.md)**. A value of NotPlanned is not acceptable because the flight had previously been planned. diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py index 81f5fb4928..fce56784c4 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py @@ -1,6 +1,19 @@ import arrow +from loguru import logger -from monitoring.monitorlib.geotemporal import resolve_volume4d +from monitoring.monitorlib.clients.flight_planning.client import ( + FlightPlannerClient, + PlanningActivityError, +) +from monitoring.monitorlib.clients.flight_planning.flight_info import ExecutionStyle +from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResult, + FlightPlanStatus, + AdvisoryInclusion, +) +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.resources.flight_planning import FlightPlannerResource from monitoring.uss_qualifier.resources.interuss.flight_authorization.definitions import ( FlightCheckTable, AcceptanceExpectation, @@ -16,10 +29,13 @@ from monitoring.uss_qualifier.scenarios.scenario import TestScenario +# Check names from documentation +_VALID_API_RESPONSE_NAME = "Valid planning response" _ACCEPT_CHECK_NAME = "Allowed flight" _REJECT_CHECK_NAME = "Disallowed flight" _CONDITIONAL_CHECK_NAME = "Required conditions" _UNCONDITIONAL_CHECK_NAME = "Disallowed conditions" +_SUCCESSFUL_CLOSURE_NAME = "Successful closure" def _get_check_by_name( @@ -30,13 +46,18 @@ def _get_check_by_name( class GeneralFlightAuthorization(TestScenario): table: FlightCheckTable + flight_planner: FlightPlannerClient + participant_id: ParticipantID def __init__( self, - table: FlightCheckTableResource, # TODO: Add new flight planner resource + table: FlightCheckTableResource, + planner: FlightPlannerResource, ): super().__init__() self.table = table.table + self.flight_planner = planner.client + self.participant_id = planner.participant_id def run(self): self.begin_test_scenario() @@ -50,7 +71,10 @@ def run(self): def _plan_flights(self): start_time = arrow.utcnow().datetime for row in self.table.rows: - checks = [] + checks = [ + _get_check_by_name(self._current_case.steps[0], name) + for name in (_VALID_API_RESPONSE_NAME, _SUCCESSFUL_CLOSURE_NAME) + ] if row.acceptance_expectation == AcceptanceExpectation.MustBeAccepted: acceptance_check = _get_check_by_name( self._current_case.steps[0], _ACCEPT_CHECK_NAME @@ -103,37 +127,159 @@ def _plan_flights(self): ) self.begin_dynamic_test_step(doc) - concrete_volumes = [resolve_volume4d(v, start_time) for v in row.volumes] + # Attempt planning action + info = row.flight_info.resolve(start_time) + with self.check(_VALID_API_RESPONSE_NAME, [self.participant_id]) as check: + try: + resp = self.flight_planner.try_plan_flight( + info, ExecutionStyle.IfAllowed + ) + except PlanningActivityError as e: + for q in e.queries: + self.record_query(q) + check.record_failed( + summary="Flight planning API request failed", + severity=Severity.High, + details=str(e), + query_timestamps=[ + q.request.initiated_at.datetime for q in e.queries + ], + ) - # TODO: Attempt to plan flight in USSs under test - self.record_note( - "flight_planning", - f"TODO: Attempt to plan flight in USSs where flight plan {row.acceptance_expectation} and conditions {row.conditions_expectation}, from {concrete_volumes[0].time_start} to {concrete_volumes[0].time_end}", - ) + logger.info(f"Recording {len(resp.queries)} queries") + for q in resp.queries: + self.record_query(q) + # Evaluate acceptance result if row.acceptance_expectation == AcceptanceExpectation.MustBeAccepted: - with self.check( - _ACCEPT_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + logger.info("Must be accepted; checking...") + with self.check(_ACCEPT_CHECK_NAME, [self.participant_id]) as check: + if resp.activity_result != PlanningActivityResult.Completed: + check.record_failed( + summary=f"Expected-accepted flight request was {resp.activity_result}", + severity=Severity.Medium, + details=f"The flight was expected to be accepted, but the activity result was indicated as {resp.activity_result}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) + if resp.flight_plan_status not in ( + FlightPlanStatus.Planned, + FlightPlanStatus.OkToFly, + ): + check.record_failed( + summary=f"Expected-accepted flight had {resp.flight_plan_status} flight plan", + severity=Severity.Medium, + details=f"The flight was expected to be accepted, but the flight plan status following the planning action was indicated as {resp.flight_plan_status}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) if row.acceptance_expectation == AcceptanceExpectation.MustBeRejected: - with self.check( - _REJECT_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + logger.info("Must be rejected; checking...") + with self.check(_REJECT_CHECK_NAME, [self.participant_id]) as check: + if resp.activity_result != PlanningActivityResult.Rejected: + check.record_failed( + summary=f"Expected-rejected flight request was {resp.activity_result}", + severity=Severity.Medium, + details=f"The flight was expected to be rejected, but the activity result was indicated as {resp.activity_result}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) + if resp.flight_plan_status != FlightPlanStatus.NotPlanned: + check.record_failed( + summary=f"Expected-accepted flight had {resp.flight_plan_status} flight plan", + severity=Severity.Medium, + details=f"The flight was expected to be rejected, but the flight plan status following the planning action was indicated as {resp.flight_plan_status}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) - # TODO: Only check conditions expectations if flight planning succeeded - if row.conditions_expectation == ConditionsExpectation.MustBePresent: - with self.check( - _CONDITIONAL_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + # Perform checks only applicable when the planning activity succeeded + if ( + resp.activity_result == PlanningActivityResult.Completed + and resp.flight_plan_status + in (FlightPlanStatus.Planned, FlightPlanStatus.OkToFly) + ): + if row.conditions_expectation == ConditionsExpectation.MustBePresent: + logger.info("Checking conditions must be present...") + with self.check( + _CONDITIONAL_CHECK_NAME, [self.participant_id] + ) as check: + if ( + resp.includes_advisories + != AdvisoryInclusion.AtLeastOneAdvisoryOrCondition + ): + check.record_failed( + summary=f"Missing expected conditions", + severity=Severity.Medium, + details=f"The flight planning activity result was expected to be accompanied by conditions/advisories, but advisory inclusion was {resp.includes_advisories}", + query_timestamps=[ + q.request.initiated_at.datetime + for q in resp.queries + ], + ) - if row.conditions_expectation == ConditionsExpectation.MustBeAbsent: + if row.conditions_expectation == ConditionsExpectation.MustBeAbsent: + logger.info("Checking conditions must be absent...") + with self.check( + _UNCONDITIONAL_CHECK_NAME, [self.participant_id] + ) as check: + if ( + resp.includes_advisories + != AdvisoryInclusion.NoAdvisoriesOrConditions + ): + check.record_failed( + summary=f"Expected-unqualified planning success was qualified by conditions", + severity=Severity.Medium, + details=f"The flight planning activity result was expected to be unqualified (accompanied by no conditions/advisories), but advisory inclusion was {resp.includes_advisories}", + query_timestamps=[ + q.request.initiated_at.datetime + for q in resp.queries + ], + ) + + # Remove flight plan if the activity resulted in a flight plan + if resp.flight_plan_status in ( + FlightPlanStatus.Planned, + FlightPlanStatus.OkToFly, + ): + logger.info("Removing flight...") + with self.check( + _VALID_API_RESPONSE_NAME, [self.participant_id] + ) as check: + try: + del_resp = self.flight_planner.try_end_flight( + resp.flight_id, ExecutionStyle.IfAllowed + ) + except PlanningActivityError as e: + for q in e.queries: + self.record_query(q) + check.record_failed( + summary="Flight planning API delete request failed", + severity=Severity.High, + details=str(e), + query_timestamps=[ + q.request.initiated_at.datetime for q in e.queries + ], + ) + for q in del_resp.queries: + self.record_query(q) with self.check( - _UNCONDITIONAL_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + _SUCCESSFUL_CLOSURE_NAME, [self.participant_id] + ) as check: + if del_resp.flight_plan_status != FlightPlanStatus.Closed: + check.record_failed( + summary="Could not close flight plan successfully", + severity=Severity.High, + details=f"Expected flight plan status to be Closed after request to end flight, but status was instead {del_resp.flight_plan_status}", + query_timestamps=[ + q.request.initiated_at.datetime + for q in del_resp.queries + ], + ) self.end_test_step() diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json new file mode 100644 index 0000000000..24ad42f3dc --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json @@ -0,0 +1,18 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Information provided about a flight plan that is necessary for ASTM F3548-21.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info.ASTMF354821OpIntentInformation, as defined in monitoring/monitorlib/clients/flight_planning/flight_info.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "priority": { + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json new file mode 100644 index 0000000000..2d8a22cef9 --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json @@ -0,0 +1,98 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "The details of a UAS flight authorization request, as received from the user.\n\nNote that a full description of a flight authorisation must include mandatory information required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664 for an UAS flight authorisation request. Reference: https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:32021R0664&from=EN#d1e32-178-1\n\nmonitoring.monitorlib.clients.flight_planning.flight_info.FlightAuthorisationData, as defined in monitoring/monitorlib/clients/flight_planning/flight_info.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "connectivity_methods": { + "description": "Connectivity methods. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 7.", + "items": { + "type": "string" + }, + "type": "array" + }, + "emergency_procedure_url": { + "description": "The URL at which the applicable emergency procedure in case of a loss of command and control link may be retrieved. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 9.", + "type": "string" + }, + "endurance_minutes": { + "description": "Endurance of the UAS. This is expressed in minutes. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 8.", + "type": "integer" + }, + "identification_technologies": { + "description": "Technology used to identify the UAS. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 6.", + "items": { + "type": "string" + }, + "type": "array" + }, + "operation_category": { + "description": "Category of UAS operation (\u2018open\u2019, \u2018specific\u2019, \u2018certified\u2019) as defined in COMMISSION DELEGATED REGULATION (EU) 2019/945. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.", + "enum": [ + "Unknown", + "Open", + "Specific", + "Certified" + ], + "type": "string" + }, + "operation_mode": { + "enum": [ + "Undeclared", + "Vlos", + "Bvlos" + ], + "type": "string" + }, + "operator_id": { + "description": "Registration number of the UAS operator.\nThe format is defined in EASA Easy Access Rules for Unmanned Aircraft Systems GM1 to AMC1\nArticle 14(6) Registration of UAS operators and \u2018certified\u2019 UAS.\nRequired by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 10.", + "type": "string" + }, + "uas_class": { + "enum": [ + "Other", + "C0", + "C1", + "C2", + "C3", + "C4", + "C5", + "C6" + ], + "type": "string" + }, + "uas_id": { + "description": "When applicable, the registration number of the unmanned aircraft.\nThis is expressed using the nationality and registration mark of the unmanned aircraft in\nline with ICAO Annex 7.\nSpecified by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 10.", + "type": [ + "string", + "null" + ] + }, + "uas_serial_number": { + "description": "Unique serial number of the unmanned aircraft or, if the unmanned aircraft is privately built, the unique serial number of the add-on. This is expressed in the ANSI/CTA-2063 Physical Serial Number format. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 1.", + "type": "string" + }, + "uas_type_certificate": { + "description": "Provisional field. Not applicable as of September 2021. Required only if `uas_class` is set to `other` by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "connectivity_methods", + "emergency_procedure_url", + "endurance_minutes", + "identification_technologies", + "operation_category", + "operation_mode", + "operator_id", + "uas_class", + "uas_serial_number" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json new file mode 100644 index 0000000000..99a396c9fa --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json @@ -0,0 +1,102 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Information about a flight necessary to plan successfully using the RPAS Platform Operating Rules version 2.6.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info.RPAS26FlightDetails, as defined in monitoring/monitorlib/clients/flight_planning/flight_info.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "aircraft_type": { + "description": "Type of vehicle being used as per ASTM F3411-22a.", + "enum": [ + "NotDeclared", + "Aeroplane", + "Helicopter", + "Gyroplane", + "HybridLift", + "Ornithopter", + "Glider", + "Kite", + "FreeBalloon", + "CaptiveBalloon", + "Airship", + "FreeFallOrParachute", + "Rocket", + "TetheredPoweredAircraft", + "GroundObstacle", + "Other" + ], + "type": [ + "string", + "null" + ] + }, + "flight_profile": { + "description": "Type of flight profile.", + "enum": [ + "AutomatedGrid", + "AutomatedWaypoint", + "Manual" + ], + "type": [ + "string", + "null" + ] + }, + "operator_number": { + "description": "Operator number.", + "type": [ + "string", + "null" + ] + }, + "operator_type": { + "description": "The type of operator.", + "enum": [ + "Recreational", + "CommercialExcluded", + "ReOC" + ], + "type": [ + "string", + "null" + ] + }, + "pilot_license_number": { + "description": "License number for the pilot.", + "type": [ + "string", + "null" + ] + }, + "pilot_phone_number": { + "description": "Contact phone number for the pilot.", + "type": [ + "string", + "null" + ] + }, + "uas_registration_numbers": { + "description": "The list of UAS/drone registration numbers that will be operated during the operation.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "uas_serial_numbers": { + "description": "The list of UAS/drone serial numbers that will be operated during the operation.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json new file mode 100644 index 0000000000..728cb5af9d --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json @@ -0,0 +1,41 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Template to provide (at runtime) basic information about a flight plan that an operator and/or UAS can be expected to provide in most flight planning scenarios.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info_template.BasicFlightPlanInformationTemplate, as defined in monitoring/monitorlib/clients/flight_planning/flight_info_template.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "area": { + "description": "User intends to or may fly anywhere in this entire area.", + "items": { + "$ref": "../../../geotemporal/Volume4DTemplate.json" + }, + "type": "array" + }, + "uas_state": { + "description": "State of the user's UAS associated with this flight plan.", + "enum": [ + "Nominal", + "OffNominal", + "Contingent" + ], + "type": "string" + }, + "usage_state": { + "description": "User's current usage of the airspace specified in the flight plan.", + "enum": [ + "Planned", + "InUse" + ], + "type": "string" + } + }, + "required": [ + "area", + "uas_state", + "usage_state" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json new file mode 100644 index 0000000000..c44de2b563 --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json @@ -0,0 +1,55 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Template to provide (at runtime) details of user's intent to create or modify a flight plan.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info_template.FlightInfoTemplate, as defined in monitoring/monitorlib/clients/flight_planning/flight_info_template.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "additional_information": { + "description": "Any information relevant to a particular jurisdiction or use case not described in the standard schema. The keys and values must be agreed upon between the test designers and USSs under test.", + "type": [ + "object", + "null" + ] + }, + "astm_f3548_21": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../flight_info/ASTMF354821OpIntentInformation.json" + } + ] + }, + "basic_information": { + "$ref": "BasicFlightPlanInformationTemplate.json" + }, + "rpas_operating_rules_2_6": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../flight_info/RPAS26FlightDetails.json" + } + ] + }, + "uspace_flight_authorisation": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../flight_info/FlightAuthorisationData.json" + } + ] + } + }, + "required": [ + "basic_information" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/fetch/Query.json b/schemas/monitoring/monitorlib/fetch/Query.json index 0255097971..d69788f3cf 100644 --- a/schemas/monitoring/monitorlib/fetch/Query.json +++ b/schemas/monitoring/monitorlib/fetch/Query.json @@ -45,7 +45,11 @@ "astm.f3548.v21.uss.getConstraintDetails", "astm.f3548.v21.uss.notifyConstraintDetailsChanged", "astm.f3548.v21.uss.makeUssReport", - "interuss.automated_testing.versioning.GetVersion" + "interuss.automated_testing.versioning.GetVersion", + "interuss.automated_testing.flight_planning.v1.GetStatus", + "interuss.automated_testing.flight_planning.v1.ClearArea", + "interuss.automated_testing.flight_planning.v1.UpsertFlightPlan", + "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" ], "type": [ "string", diff --git a/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json b/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json index 7ee3137a1d..7c40f148cb 100644 --- a/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json +++ b/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json @@ -16,10 +16,6 @@ ], "type": "string" }, - "additional_information": { - "description": "Any additional information that should be provided to a USS planning the flight.\n\nFormat is agreed upon between test designer and USSs.", - "type": "object" - }, "conditions_expectation": { "description": "Expected conditions/advisories produced when authorizing a flight as described.", "enum": [ @@ -37,27 +33,23 @@ "description": "Unique (within table) test step/row identifier.", "type": "string" }, + "flight_info": { + "$ref": "../../../../../monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json", + "description": "Information about the flight, as a user would provide it to the USS." + }, "requirement_ids": { "description": "Jurisdictional identifiers of the requirements this test step is evaluating.", "items": { "type": "string" }, "type": "array" - }, - "volumes": { - "description": "Spatial and temporal definition of the areas the virtual user intends to fly in.\n\nA service provider is expected to authorizing a flight covering the entire area specified and for any of the entire time specified.", - "items": { - "$ref": "../../../../../monitorlib/geotemporal/Volume4DTemplate.json" - }, - "type": "array" } }, "required": [ - "additional_information", "description", "flight_check_id", - "requirement_ids", - "volumes" + "flight_info", + "requirement_ids" ], "type": "object" } \ No newline at end of file