diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py index b6462fe91..08d420b5e 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py +++ b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py @@ -6,7 +6,13 @@ from monitoring.uss_qualifier.reports.report import TestSuiteActionReport from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource -from monitoring.uss_qualifier.resources.resource import ResourceType +from monitoring.uss_qualifier.resources.flight_planning.flight_planners import ( + FlightPlannerCombinationSelectorResource, +) +from monitoring.uss_qualifier.resources.resource import ( + ResourceType, + make_child_resources, +) from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration from monitoring.uss_qualifier.suites.suite import ( @@ -23,17 +29,12 @@ class FlightPlannerCombinationsSpecification(ImplicitDict): flight_planners_source: ResourceID """ID of the resource providing all available flight planners""" + combination_selector_source: Optional[ResourceID] = None + """If specified and contained in the provided resources, the resource containing a FlightPlannerCombinationSelectorResource to select only a subset of combinations""" + roles: int """Number of flight planners to make available to the action, via whichever resource ID is mapped to the parent `flight_planners_source`""" - resources: Dict[ResourceID, ResourceID] - """Mapping of the ID a resource will be known by in the child action -> the ID a resource is known by in the parent action generator. - - The child action resource is supplied by the parent action generator . - - Resources not included in this field or in `roles` will not be available to the child action. - """ - class FlightPlannerCombinations( ActionGenerator[FlightPlannerCombinationsSpecification] @@ -49,7 +50,7 @@ def __init__( ): if specification.flight_planners_source not in resources: raise ValueError( - f"Resource ID {specification.flight_planners_source} was not present in the available resource pool" + f"Resource ID {specification.flight_planners_source} specified as `flight_planners_source` was not present in the available resource pool" ) flight_planners_resource: FlightPlannersResource = resources[ specification.flight_planners_source @@ -60,21 +61,42 @@ def __init__( ) flight_planners = flight_planners_resource.flight_planners + if ( + specification.combination_selector_source is not None + and specification.combination_selector_source in resources + ): + combination_selector = resources[specification.combination_selector_source] + if not isinstance( + combination_selector, FlightPlannerCombinationSelectorResource + ): + raise ValueError( + f"Expected resource ID {specification.combination_selector_source} to be a {fullname(FlightPlannerCombinationSelectorResource)} but it was a {fullname(combination_selector.__class__)} instead" + ) + else: + combination_selector = None + self._actions = [] role_assignments = [0] * specification.roles while True: - modified_parent_resources = {k: v for k, v in resources.items()} - modified_parent_resources[ - specification.flight_planners_source - ] = flight_planners_resource.make_subset(role_assignments) - resources_for_child = { - child_resource_id: modified_parent_resources[parent_resource_id] - for child_resource_id, parent_resource_id in specification.resources.items() - } - self._actions.append( - TestSuiteAction(specification.action_to_repeat, resources_for_child) + flight_planners_combination = flight_planners_resource.make_subset( + role_assignments ) + if ( + combination_selector is None + or combination_selector.is_valid_combination( + flight_planners_combination + ) + ): + modified_resources = {k: v for k, v in resources.items()} + modified_resources[ + specification.flight_planners_source + ] = flight_planners_combination + + self._actions.append( + TestSuiteAction(specification.action_to_repeat, modified_resources) + ) + index_to_increment = len(role_assignments) - 1 while index_to_increment >= 0: role_assignments[index_to_increment] += 1 diff --git a/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml b/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml new file mode 100644 index 000000000..6c727cf88 --- /dev/null +++ b/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml @@ -0,0 +1,56 @@ +resources: + resource_declarations: + "$ref": ../../resources.yaml#/common + flight_planners: + resource_type: resources.flight_planning.FlightPlannersResource + dependencies: + auth_adapter: utm_auth + specification: + flight_planners: + - participant_id: uss1 + injection_base_url: http://host.docker.internal:8074/scdsc + - participant_id: uss2 + injection_base_url: http://host.docker.internal:8074/scdsc + - participant_id: mock_uss + injection_base_url: http://host.docker.internal:8074/scdsc + combination_selector: + resource_type: resources.flight_planning.FlightPlannerCombinationSelectorResource + specification: + must_include: + - mock_uss + maximum_roles: + mock_uss: 1 + conflicting_flights: + resource_type: resources.flight_planning.FlightIntentsResource + specification: + planning_time: '0:05:00' + file_source: file://./test_data/che/flight_intents/conflicting_flights.json + priority_preemption_flights: + resource_type: resources.flight_planning.FlightIntentsResource + specification: + planning_time: '0:05:00' + file_source: test_data.che.flight_intents.priority_preemption + dss: + resource_type: resources.astm.f3548.v21.DSSInstanceResource + dependencies: + auth_adapter: utm_auth + specification: + participant_id: uss1 + base_url: http://host.docker.internal:8082 + mock_uss: + resource_type: resources.interuss.MockUSSResource + dependencies: + auth_adapter: utm_auth + specification: + participant_id: mock_uss + mock_uss_base_url: http://host.docker.internal:8074 + +test_suite: + suite_type: suites.faa.uft.message_signing + resources: + mock_uss: mock_uss + flight_planners: flight_planners + combination_selector: combination_selector + conflicting_flights: conflicting_flights + priority_preemption_flights: priority_preemption_flights + dss: dss diff --git a/monitoring/uss_qualifier/resources/flight_planning/__init__.py b/monitoring/uss_qualifier/resources/flight_planning/__init__.py index 4f555f241..254cef3f3 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/__init__.py +++ b/monitoring/uss_qualifier/resources/flight_planning/__init__.py @@ -1,2 +1,5 @@ -from .flight_planners import FlightPlannersResource +from .flight_planners import ( + FlightPlannersResource, + FlightPlannerCombinationSelectorResource, +) from .flight_intents_resource import FlightIntentsResource diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py index b709217b4..403669daa 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py @@ -1,6 +1,7 @@ -from typing import List, Iterable +from typing import List, Iterable, Dict from implicitdict import ImplicitDict +from monitoring.uss_qualifier.reports.report import ParticipantID from monitoring.uss_qualifier.resources.resource import Resource from monitoring.uss_qualifier.resources.communications import AuthAdapterResource @@ -32,3 +33,43 @@ def make_subset(self, select_indices: Iterable[int]) -> "FlightPlannersResource" subset_resource = FlightPlannersResource.__new__(FlightPlannersResource) subset_resource.flight_planners = subset return subset_resource + + +class FlightPlannerCombinationSelectorSpecification(ImplicitDict): + must_include: List[ParticipantID] + """The set of flight planners which must be included in every combination""" + + maximum_roles: Dict[ParticipantID, int] + """Maximum number of roles a particular participant may fill in any given combination""" + + +class FlightPlannerCombinationSelectorResource( + Resource[FlightPlannerCombinationSelectorSpecification] +): + _specification: FlightPlannerCombinationSelectorSpecification + + def __init__(self, specification: FlightPlannerCombinationSelectorSpecification): + self._specification = specification + + def is_valid_combination(self, flight_planners: FlightPlannersResource): + participants = [p.participant_id for p in flight_planners.flight_planners] + + accept_combination = True + + # Apply must_include criteria + for required_participant in self._specification.must_include: + if required_participant not in participants: + accept_combination = False + break + + # Apply maximum_roles criteria + for limited_participant, max_count in self._specification.maximum_roles.items(): + count = sum( + (1 if participant == limited_participant else 0) + for participant in participants + ) + if count > max_count: + accept_combination = False + break + + return accept_combination diff --git a/monitoring/uss_qualifier/resources/interuss/__init__.py b/monitoring/uss_qualifier/resources/interuss/__init__.py new file mode 100644 index 000000000..0cc63a6db --- /dev/null +++ b/monitoring/uss_qualifier/resources/interuss/__init__.py @@ -0,0 +1 @@ +from .mock_uss import MockUSSResource diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss.py b/monitoring/uss_qualifier/resources/interuss/mock_uss.py new file mode 100644 index 000000000..759965276 --- /dev/null +++ b/monitoring/uss_qualifier/resources/interuss/mock_uss.py @@ -0,0 +1,60 @@ +import arrow + +from implicitdict import ImplicitDict +from monitoring.monitorlib import fetch +from monitoring.monitorlib.infrastructure import AuthAdapter, UTMClientSession +from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( + SCOPE_SCD_QUALIFIER_INJECT, +) +from monitoring.uss_qualifier.reports.report import ParticipantID +from monitoring.uss_qualifier.resources.communications import AuthAdapterResource +from monitoring.uss_qualifier.resources.resource import Resource + + +class MockUSSClient(object): + """Means to communicate with an InterUSS mock_uss instance""" + + def __init__( + self, + participant_id: str, + base_url: str, + auth_adapter: AuthAdapter, + ): + self.session = UTMClientSession(base_url, auth_adapter) + self.participant_id = participant_id + + def get_status(self) -> fetch.Query: + initiated_at = arrow.utcnow().datetime + resp = self.session.get("/scdsc/v1/status", scope=SCOPE_SCD_QUALIFIER_INJECT) + return fetch.describe_query(resp, initiated_at) + + # TODO: Add other methods to interact with the mock USS in other ways (like starting/stopping message signing data collection) + + +class MockUSSSpecification(ImplicitDict): + mock_uss_base_url: str + """The base URL for the mock USS. + + If the mock USS had scdsc enabled, for instance, then these URLs would be + valid: + * /mock/scd/uss/v1/reports + * /scdsc/v1/status + """ + + participant_id: ParticipantID + """Test participant responsible for this mock USS.""" + + +class MockUSSResource(Resource[MockUSSSpecification]): + mock_uss: MockUSSClient + + def __init__( + self, + specification: MockUSSSpecification, + auth_adapter: AuthAdapterResource, + ): + self.mock_uss = MockUSSClient( + specification.participant_id, + specification.mock_uss_base_url, + auth_adapter.adapter, + ) diff --git a/monitoring/uss_qualifier/resources/resource.py b/monitoring/uss_qualifier/resources/resource.py index c1fbb280e..90d9c38bd 100644 --- a/monitoring/uss_qualifier/resources/resource.py +++ b/monitoring/uss_qualifier/resources/resource.py @@ -117,3 +117,22 @@ def _make_resource( ) return resource_type(**constructor_args) + + +def make_child_resources( + parent_resources: Dict[ResourceID, ResourceType], + child_resource_map: Dict[ResourceID, ResourceID], + subject: str, +) -> Dict[ResourceID, ResourceType]: + child_resources = {} + for child_id, parent_id in child_resource_map.items(): + is_optional = parent_id.endswith("?") + if is_optional: + parent_id = parent_id[:-1] + if parent_id in parent_resources: + child_resources[child_id] = parent_resources[parent_id] + elif not is_optional: + raise ValueError( + f'{subject} could not find required resource ID "{parent_id}" used to populate child resource ID "{child_id}"' + ) + return child_resources diff --git a/monitoring/uss_qualifier/scenarios/faa/__init__.py b/monitoring/uss_qualifier/scenarios/faa/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/__init__.py b/monitoring/uss_qualifier/scenarios/faa/uft/__init__.py new file mode 100644 index 000000000..32fc7ce37 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/faa/uft/__init__.py @@ -0,0 +1,2 @@ +from .message_signing_start import StartMessageSigningReport +from .message_signing_finalize import FinalizeMessageSigningReport diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md new file mode 100644 index 000000000..f3c4e1aa1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md @@ -0,0 +1,17 @@ +# Finalize message signing test scenario + +This test scenario instructs a mock USS to finalize a message signing report from captured data. + +## Resources + +### mock_uss + +The means to communicate with the mock USS that has been collecting message signing data. + +## Finalize message signing test case + +### Signal mock USS test step + +#### Successful finalization check + +If the mock USS doesn't finalize the message signing report successfully, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py new file mode 100644 index 000000000..2f837bee6 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py @@ -0,0 +1,40 @@ +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.interuss.mock_uss import ( + MockUSSResource, + MockUSSClient, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class FinalizeMessageSigningReport(TestScenario): + _mock_uss: MockUSSClient + + def __init__(self, mock_uss: MockUSSResource): + super().__init__() + self._mock_uss = mock_uss.mock_uss + + def run(self): + self.begin_test_scenario() + + self.begin_test_case("Finalize message signing") + + self.begin_test_step("Signal mock USS") + + # TODO: Add call to mock USS to finalize message signing report + with self.check( + "Successful finalization", participants=[self._mock_uss.participant_id] + ) as check: + if False: # TODO: Insert appropriate check + check.record_failed( + summary="Failed to finalize message signing report", + details="TODO", + severity=Severity.High, + query_timestamps=[], + ) + return + + self.end_test_step() # Signal mock USS + + self.end_test_case() # Start message signing + + self.end_test_scenario() diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md new file mode 100644 index 000000000..2c1fe50e1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md @@ -0,0 +1,27 @@ +# Start message signing test scenario + +This test scenario instructs a mock USS to begin capturing message signing data. + +## Resources + +### mock_uss + +The means to communicate with the mock USS that will collect message signing data. + +## Start message signing test case + +### Check mock USS readiness test step + +#### Status ok check + +If the mock USS doesn't respond properly to a request for its status, this check will fail. + +#### Ready check + +If the mock USS doesn't indicate Ready for its scd functionality, this check will fail. + +### Signal mock USS test step + +#### Successful start check + +If the mock USS doesn't start capturing message signing data successfully, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py new file mode 100644 index 000000000..19229a5da --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py @@ -0,0 +1,70 @@ +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.interuss.mock_uss import ( + MockUSSResource, + MockUSSClient, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class StartMessageSigningReport(TestScenario): + _mock_uss: MockUSSClient + + def __init__(self, mock_uss: MockUSSResource): + super().__init__() + self._mock_uss = mock_uss.mock_uss + + def run(self): + self.begin_test_scenario() + + self.begin_test_case("Start message signing") + + self.begin_test_step("Check mock USS readiness") + + query = self._mock_uss.get_status() + self.record_query(query) + + with self.check( + "Status ok", participants=[self._mock_uss.participant_id] + ) as check: + if query.status_code != 200: + check.record_failed( + summary="Failed to get status from mock USS", + details=f"Status code {query.status_code}", + severity=Severity.High, + query_timestamps=[query.request.timestamp], + ) + return # Return if this scenario cannot continue + + with self.check("Ready", participants=[self._mock_uss.participant_id]) as check: + status = query.response.get("json", {}).get("status", "") + if status != "Ready": + check.record_failed( + summary="Mock USS SCD functionality not ready", + details=f"Status indicated as: {status}", + severity=Severity.High, + query_timestamps=[query.request.timestamp], + ) + return + + self.end_test_step() # Check mock USS readiness + + self.begin_test_step("Signal mock USS") + + # TODO: Add call to mock USS to start message signing report + with self.check( + "Successful start", participants=[self._mock_uss.participant_id] + ) as check: + if False: # TODO: Insert appropriate check + check.record_failed( + summary="Failed to start message signing report", + details="TODO", + severity=Severity.High, + query_timestamps=[], + ) + return + + self.end_test_step() # Signal mock USS + + self.end_test_case() # Start message signing + + self.end_test_scenario() diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index 19ab7ff8c..75b5c500d 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -4,18 +4,17 @@ resources: dss: resources.astm.f3548.v21.DSSInstanceResource conflicting_flights: resources.flight_planning.FlightIntentsResource priority_preemption_flights: resources.flight_planning.FlightIntentsResource + nominal_planning_selector: resources.flight_planning.FlightPlannerCombinationSelectorResource? + priority_planning_selector: resources.flight_planning.FlightPlannerCombinationSelectorResource? actions: - action_generator: generator_type: action_generators.flight_planning.FlightPlannerCombinations resources: flight_planners: flight_planners + nominal_planning_selector: nominal_planning_selector? conflicting_flights: conflicting_flights dss: dss specification: - resources: - flight_planners: flight_planners - conflicting_flights: conflicting_flights - dss: dss action_to_repeat: test_scenario: scenario_type: scenarios.astm.utm.NominalPlanning @@ -24,6 +23,7 @@ actions: flight_planners: flight_planners dss: dss on_failure: Continue + combination_selector_source: nominal_planning_selector flight_planners_source: flight_planners roles: 2 on_failure: Continue @@ -31,13 +31,10 @@ actions: generator_type: action_generators.flight_planning.FlightPlannerCombinations resources: flight_planners: flight_planners + priority_planning_selector: priority_planning_selector? priority_preemption_flights: priority_preemption_flights dss: dss specification: - resources: - flight_planners: flight_planners - priority_preemption_flights: priority_preemption_flights - dss: dss action_to_repeat: test_scenario: scenario_type: scenarios.astm.utm.NominalPlanningPriority @@ -46,6 +43,7 @@ actions: flight_planners: flight_planners dss: dss on_failure: Continue + combination_selector_source: priority_planning_selector flight_planners_source: flight_planners roles: 2 on_failure: Continue diff --git a/monitoring/uss_qualifier/suites/definitions.py b/monitoring/uss_qualifier/suites/definitions.py index 1e13be257..3fa012339 100644 --- a/monitoring/uss_qualifier/suites/definitions.py +++ b/monitoring/uss_qualifier/suites/definitions.py @@ -38,9 +38,13 @@ class ActionGeneratorDefinition(ImplicitDict): """Specification of action generator; format is the ActionGeneratorSpecificationType that corresponds to the `generator_type`""" resources: Dict[ResourceID, ResourceID] - """Mapping of the ID a resource will be known by in the action generator -> the ID a resource is known by in the test suite. + """Mapping of the ID a resource will be known by in the child action -> the ID a resource is known by in the parent test suite. - The action generator resource is supplied by the test suite's resource . + The child action resource ID is supplied by the parent test suite resource ID . + + Resources not included in this field will not be available to the child action. + + If the parent resource ID is suffixed with ? then the resource will not be required (and will not be populated for the child action when not present in the parent) """ diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml b/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml new file mode 100644 index 000000000..33c39c861 --- /dev/null +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml @@ -0,0 +1,29 @@ +name: UFT message signing +resources: + mock_uss: resources.interuss.MockUSSResource + flight_planners: resources.flight_planning.FlightPlannersResource + combination_selector: resources.flight_planning.FlightPlannerCombinationSelectorResource + dss: resources.astm.f3548.v21.DSSInstanceResource + conflicting_flights: resources.flight_planning.FlightIntentsResource + priority_preemption_flights: resources.flight_planning.FlightIntentsResource +actions: +- test_scenario: + scenario_type: scenarios.faa.uft.StartMessageSigningReport + resources: + mock_uss: mock_uss + on_failure: Abort +- test_suite: + suite_type: suites.astm.utm.f3548_21 + resources: + conflicting_flights: conflicting_flights + priority_preemption_flights: priority_preemption_flights + flight_planners: flight_planners + nominal_planning_selector: combination_selector + priority_planning_selector: combination_selector + dss: dss + on_failure: Continue +- test_scenario: + scenario_type: scenarios.faa.uft.FinalizeMessageSigningReport + resources: + mock_uss: mock_uss + on_failure: Continue diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index 6da2a047e..1b5c8638e 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -21,7 +21,10 @@ TestSuiteActionReport, ) from monitoring.uss_qualifier.resources.definitions import ResourceID -from monitoring.uss_qualifier.resources.resource import ResourceType +from monitoring.uss_qualifier.resources.resource import ( + ResourceType, + make_child_resources, +) from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.suites.definitions import ( TestSuiteActionDeclaration, @@ -52,16 +55,11 @@ def __init__( resources: Dict[ResourceID, ResourceType], ): self.declaration = action - - for parent_resource_id in action.get_resource_links().values(): - if parent_resource_id not in resources: - raise ValueError( - f"Test suite action to run {action.get_action_type()} {action.get_child_type()} is missing resource {parent_resource_id} from the parent test suite" - ) - resources_for_child = { - child_resource_id: resources[parent_resource_id] - for child_resource_id, parent_resource_id in action.get_resource_links().items() - } + resources_for_child = make_child_resources( + resources, + action.get_resource_links(), + f"Test suite action to run {action.get_action_type()} {action.get_child_type()}", + ) action_type = action.get_action_type() if action_type == ActionType.TestScenario: @@ -145,11 +143,16 @@ def __init__( for local_resource_id, parent_resource_id in declaration.resources.items() } for resource_id, resource_type in self.definition.resources.items(): - if resource_id not in local_resources: + is_optional = resource_type.endswith("?") + if is_optional: + resource_type = resource_type[:-1] + if not is_optional and resource_id not in local_resources: raise ValueError( f'Test suite "{self.definition.name}" is missing resource {resource_id} ({resource_type})' ) - if not local_resources[resource_id].is_type(resource_type): + if resource_id in local_resources and not local_resources[ + resource_id + ].is_type(resource_type): raise ValueError( f'Test suite "{self.definition.name}" expected resource {resource_id} to be {resource_type}, but instead it was provided {fullname(resources[resource_id].__class__)}' )