diff --git a/monitoring/mock_uss/flight_planning/routes.py b/monitoring/mock_uss/flight_planning/routes.py index 79067624df..6dc8645bfd 100644 --- a/monitoring/mock_uss/flight_planning/routes.py +++ b/monitoring/mock_uss/flight_planning/routes.py @@ -135,7 +135,7 @@ def flight_planning_v1_clear_area() -> Tuple[str, int]: resp = api.ClearAreaResponse( outcome=api.ClearAreaOutcome( success=clear_resp.success, - message="See `details`", + message="See `details` field in response for more information", details=clear_resp, ) ) diff --git a/monitoring/mock_uss/scd_injection/routes_injection.py b/monitoring/mock_uss/scd_injection/routes_injection.py index 58e8fb6b86..a00bf12d79 100644 --- a/monitoring/mock_uss/scd_injection/routes_injection.py +++ b/monitoring/mock_uss/scd_injection/routes_injection.py @@ -327,7 +327,7 @@ def scdsc_clear_area() -> Tuple[str, int]: resp = scd_api.ClearAreaResponse( outcome=ClearAreaOutcome( success=clear_resp.success, - message="See `details` field for more information", + message="See `details` field in response for more information", timestamp=StringBasedDateTime(datetime.utcnow()), ), ) diff --git a/monitoring/monitorlib/clients/flight_planning/client_v1.py b/monitoring/monitorlib/clients/flight_planning/client_v1.py index ce91c33182..5e2c5be4fb 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_v1.py +++ b/monitoring/monitorlib/clients/flight_planning/client_v1.py @@ -212,6 +212,6 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: if resp.outcome.success: errors = None else: - errors = [f"[{resp.outcome.timestamp}]: {resp.outcome.message}"] + errors = [resp.outcome.message] return TestPreparationActivityResponse(errors=errors, queries=[query]) diff --git a/monitoring/monitorlib/inspection.py b/monitoring/monitorlib/inspection.py index 8f980daac8..d6408f04ba 100644 --- a/monitoring/monitorlib/inspection.py +++ b/monitoring/monitorlib/inspection.py @@ -32,5 +32,11 @@ def get_module_object_by_name(parent_module, object_name: str): def fullname(class_type: Type) -> str: module = class_type.__module__ if module == "builtins": - return class_type.__qualname__ # avoid outputs like 'builtins.str' - return module + "." + class_type.__qualname__ + if hasattr(class_type, "__qualname__"): + return class_type.__qualname__ # avoid outputs like 'builtins.str' + else: + return str(class_type) + if hasattr(class_type, "__qualname__"): + return module + "." + class_type.__qualname__ + else: + return str(class_type) diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md index b401fdad94..492be370a9 100644 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md @@ -10,6 +10,10 @@ When a USS implements the [InterUSS flight_planning automated testing API](https A USS must implement the endpoints defined in the API, accept requests in the data format prescribed in the API, and respond in the data format prescribed in the API. If there is a problem using the API such as a connection error, invalid response code, or invalid data, the USS will have failed to meet this requirement. +### Readiness + +A USS must implement the readiness endpoint defined in the API and then respond that it is ready to respond with an appropriate API version. + ### ClearArea In order to conduct automated tests effectively, the USS must remove all of their existing flights from a particular area when instructed by the test director. This is not an action performed on behalf of an emulated user, but rather an action performed in any way appropriate to support automated testing -- therefore, fulfilling this request may cause actions on the implementing USS's system that no normal user would be able to perform. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py index 8a5637e620..fee5010cb1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/__init__.py @@ -7,3 +7,4 @@ ) from .dss_interoperability import DSSInteroperability from .aggregate_checks import AggregateChecks +from .prep_planners import PrepareFlightPlanners diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md new file mode 100644 index 0000000000..bff29b4a02 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.md @@ -0,0 +1,69 @@ +# ASTM F3548 flight planners preparation test scenario + +## Description + +This scenario prepares flight planner systems for execution of controlled test scenarios by checking planner systems' readiness and having them remove any existing flights that may already be in the test area. + +## Resources + +### flight_planners + +FlightPlannersResource listing all USSs undergoing planning tests so that they can be checked for readiness and instructed to remove any existing flights from the area in this scenario. + +### dss + +DSSInstanceResource to check for lingering operational intents after the area has been cleared. + +### flight_intents + +FlightIntentsResource containing flight intents that will be used in subsequent tests, so all planners should be instructed to clear any area involved with any of these intents of flights it manages. + +### flight_intents2 + +(Optional) If more than one FlightIntentsResource will be used in subsequent tests, additional intents may be specified with this resource. + +### flight_intents3 + +(Optional) If more than one FlightIntentsResource will be used in subsequent tests, additional intents may be specified with this resource. + +### flight_intents4 + +(Optional) If more than one FlightIntentsResource will be used in subsequent tests, additional intents may be specified with this resource. + +## Preparation test case + +### Check for flight planning readiness test step + +All USSs are queried for their readiness to ensure later tests can proceed. + +#### ⚠️ Valid response to readiness query check + +**[interuss.automated_testing.flight_planning.ImplementAPI](../../../requirements/interuss/automated_testing/flight_planning.md)** + +#### ⚠️ Flight planning USS ready check + +This readiness indicates the USS's ability to inject test data, so if this check fails, not only has the USS not met **[interuss.automated_testing.flight_planning.Readiness](../../../requirements/interuss/automated_testing/flight_planning.md)**, but it also does not meet **[astm.f3548.v21.GEN0310](../../../requirements/astm/f3548/v21.md)**. + +### Area clearing test step + +All USSs are requested to remove all flights from the area under test. + +#### ⚠️ Valid response to clearing query check + +**[interuss.automated_testing.flight_planning.ImplementAPI](../../../requirements/interuss/automated_testing/flight_planning.md)** + +#### ⚠️ Area cleared successfully check + +**[interuss.automated_testing.flight_planning.ClearArea](../../../requirements/interuss/automated_testing/flight_planning.md)** + +### Clear area validation test step + +uss_qualifier verifies with the DSS that there are no operational intents remaining in the area + +#### 🛑 DSS responses check + +**[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)** + +#### 🛑 Area is clear check + +If operational intents remain in the 4D area(s) following the preceding area clearing, then the current state of the test environment is not suitable to conduct tests so this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py new file mode 100644 index 0000000000..f40ea87afb --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/prep_planners.py @@ -0,0 +1,88 @@ +from typing import Optional + +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource +from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance +from monitoring.uss_qualifier.resources.flight_planning import ( + FlightPlannersResource, + FlightIntentsResource, +) +from monitoring.uss_qualifier.scenarios.flight_planning.prep_planners import ( + PrepareFlightPlanners as GenericPrepareFlightPlanners, +) + + +class PrepareFlightPlanners(GenericPrepareFlightPlanners): + dss: DSSInstance + + def __init__( + self, + flight_planners: FlightPlannersResource, + dss: DSSInstanceResource, + flight_intents: FlightIntentsResource, + flight_intents2: Optional[FlightIntentsResource] = None, + flight_intents3: Optional[FlightIntentsResource] = None, + flight_intents4: Optional[FlightIntentsResource] = None, + ): + super(PrepareFlightPlanners, self).__init__( + flight_planners, + flight_intents, + flight_intents2, + flight_intents3, + flight_intents4, + ) + self.dss = dss.dss + + def run(self, context): + self.begin_test_scenario(context) + self.begin_test_case("Preparation") + + self.begin_test_step("Check for flight planning readiness") + self._check_readiness() + self.end_test_step() + + self.begin_test_step("Area clearing") + self._clear_area() + self.end_test_step() + + self.begin_test_step("Clear area validation") + self._validate_clear_area() + self.end_test_step() + + self.end_test_case() + self.end_test_scenario() + + def _validate_clear_area(self): + for area in self.areas: + with self.check("DSS responses", [self.dss.participant_id]) as check: + try: + op_intents, query = self.dss.find_op_intent(area.to_f3548v21()) + except ValueError as e: + check.record_failed( + summary="Error parsing DSS response", + details=str(e), + severity=Severity.High, + ) + self.record_query(query) + if op_intents is None: + check.record_failed( + summary="Error querying DSS for operational intents", + details="See query", + severity=Severity.High, + query_timestamps=[query.request.timestamp], + ) + with self.check("Area is clear") as check: + if op_intents: + summary = f"{len(op_intents)} operational intent{'s' if len(op_intents) > 1 else ''} found in cleared area" + details = ( + "The following operational intents were observed even after clearing the area:\n" + + "\n".join( + f"* {oi.id} managed by {oi.manager}" for oi in op_intents + ) + ) + check.record_failed( + summary=summary, + details=details, + severity=Severity.High, + query_timestamps=[query.request.timestamp], + ) diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index d90ad2cb1e..7975e576d6 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -193,7 +193,12 @@ def _get_anchors( anchors = {} if isinstance(value, marko.block.Heading): - base_anchor = "#" + text_of(value).lower().replace(" ", "-") + heading_text = text_of(value) + for s in Severity: + if heading_text.startswith(s.symbol): + heading_text = heading_text[len(s.symbol) :].lstrip() + break + base_anchor = "#" + heading_text.lower().replace(" ", "-") if base_anchor not in header_counts: anchors[value] = base_anchor else: diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/__init__.py b/monitoring/uss_qualifier/scenarios/flight_planning/__init__.py index 9ea4d68487..c6a8cfe1f5 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/__init__.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/__init__.py @@ -1 +1,2 @@ from .record_planners import RecordPlanners +from .prep_planners import PrepareFlightPlanners diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md new file mode 100644 index 0000000000..3a25e1dc85 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.md @@ -0,0 +1,53 @@ +# Generic flight planners preparation test scenario + +## Description + +This scenario prepares flight planner systems for execution of controlled test scenarios by checking planner systems' readiness and having them remove any existing flights that may already be in the test area. + +## Resources + +### flight_planners + +FlightPlannersResource listing all USSs undergoing planning tests so that they can be checked for readiness and instructed to remove any existing flights from the area in this scenario. + +### flight_intents + +FlightIntentsResource containing flight intents that will be used in subsequent tests, so all planners should be instructed to clear any area involved with any of these intents of flights it manages. + +### flight_intents2 + +(Optional) If more than one FlightIntentsResource will be used in subsequent tests, additional intents may be specified with this resource. + +### flight_intents3 + +(Optional) If more than one FlightIntentsResource will be used in subsequent tests, additional intents may be specified with this resource. + +### flight_intents4 + +(Optional) If more than one FlightIntentsResource will be used in subsequent tests, additional intents may be specified with this resource. + +## Preparation test case + +### Check for flight planning readiness test step + +All USSs are queried for their readiness to ensure later tests can proceed. + +#### ⚠️ Valid response to readiness query check + +**[interuss.automated_testing.flight_planning.ImplementAPI](../../requirements/interuss/automated_testing/flight_planning.md)** + +#### ⚠️ Flight planning USS ready check + +**[interuss.automated_testing.flight_planning.Readiness](../../requirements/interuss/automated_testing/flight_planning.md)** + +### Area clearing test step + +All USSs are requested to remove all flights from the area under test. + +#### ⚠️ Valid response to clearing query check + +**[interuss.automated_testing.flight_planning.ImplementAPI](../../requirements/interuss/automated_testing/flight_planning.md)** + +#### ⚠️ Area cleared successfully check + +**[interuss.automated_testing.flight_planning.ClearArea](../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py new file mode 100644 index 0000000000..6d3c82d43c --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prep_planners.py @@ -0,0 +1,137 @@ +from datetime import timedelta +from typing import Optional, Dict, List + +import arrow + +from monitoring.monitorlib.clients.flight_planning.client import ( + FlightPlannerClient, + PlanningActivityError, +) +from monitoring.monitorlib.geotemporal import Volume4DCollection, Volume4D +from monitoring.monitorlib.temporal import Time +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.resources.flight_planning import ( + FlightPlannersResource, + FlightIntentsResource, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + +MAX_TEST_DURATION = timedelta(minutes=15) +"""The maximum time the tests depending on the area being clear might last.""" + + +class PrepareFlightPlanners(TestScenario): + areas: List[Volume4D] + flight_planners: Dict[ParticipantID, FlightPlannerClient] + + def __init__( + self, + flight_planners: FlightPlannersResource, + flight_intents: FlightIntentsResource, + flight_intents2: Optional[FlightIntentsResource] = None, + flight_intents3: Optional[FlightIntentsResource] = None, + flight_intents4: Optional[FlightIntentsResource] = None, + ): + super().__init__() + now = Time(arrow.utcnow().datetime) + later = now.offset(MAX_TEST_DURATION) + self.areas = [] + for intents in ( + flight_intents, + flight_intents2, + flight_intents3, + flight_intents4, + ): + if intents is None: + continue + v4c = Volume4DCollection([]) + for flight_info_template in intents.get_flight_intents().values(): + v4c.extend( + flight_info_template.resolve( + start_of_test=now + ).basic_information.area + ) + v4c.extend( + flight_info_template.resolve( + start_of_test=later + ).basic_information.area + ) + self.areas.append(v4c.bounding_volume) + self.flight_planners = { + fp.participant_id: fp.client for fp in flight_planners.flight_planners + } + + def run(self, context): + self.begin_test_scenario(context) + self.begin_test_case("Preparation") + + self.begin_test_step("Check for flight planning readiness") + self._check_readiness() + self.end_test_step() + + self.begin_test_step("Area clearing") + self._clear_area() + self.end_test_step() + + self.end_test_case() + self.end_test_scenario() + + def _check_readiness(self): + for participant_id, client in self.flight_planners.items(): + with self.check( + "Valid response to readiness query", [participant_id] + ) as check: + try: + resp = client.report_readiness() + except PlanningActivityError as e: + for q in e.queries: + self.record_query(q) + check.record_failed( + summary=f"Error while determining readiness of {participant_id}", + details=str(e), + severity=Severity.Medium, + query_timestamps=[q.request.timestamp for q in e.queries], + ) + continue + for q in resp.queries: + self.record_query(q) + with self.check("Flight planning USS ready", [participant_id]) as check: + if resp.errors: + check.record_failed( + summary=f"Errors in {participant_id} readiness", + details="\n".join("* " + e for e in resp.errors), + severity=Severity.Medium, + query_timestamps=[q.request.timestamp for q in resp.queries], + ) + + def _clear_area(self): + for area in self.areas: + for participant_id, client in self.flight_planners.items(): + with self.check( + "Valid response to clearing query", [participant_id] + ) as check: + try: + resp = client.clear_area(area) + except PlanningActivityError as e: + for q in e.queries: + self.record_query(q) + check.record_failed( + summary=f"Error while instructing {participant_id} to clear area", + details=str(e), + severity=Severity.Medium, + query_timestamps=[q.request.timestamp for q in e.queries], + ) + continue + for q in resp.queries: + self.record_query(q) + with self.check("Area cleared successfully", [participant_id]) as check: + if resp.errors: + check.record_failed( + summary=f"Errors when {participant_id} was clearing the area", + details="\n".join("* " + e for e in resp.errors), + severity=Severity.Medium, + query_timestamps=[ + q.request.timestamp for q in resp.queries + ], + ) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 5f79ce6d09..a6c28839f5 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -6,15 +6,16 @@ 1. Action generator: [`action_generators.astm.f3548.ForEachDSS`](../../../action_generators/astm/f3548/for_each_dss.py) 1. Suite: [DSS testing for ASTM NetRID F3548-21](dss_probing.md) ([`suites.astm.utm.dss_probing`](dss_probing.yaml)) -2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) +2. Scenario: [ASTM F3548 flight planners preparation](../../../scenarios/astm/utm/prep_planners.md) ([`scenarios.astm.utm.PrepareFlightPlanners`](../../../scenarios/astm/utm/prep_planners.py)) 3. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) + 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) 4. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) + 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) +5. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) -5. Scenario: [ASTM F3548 UTM aggregate checks](../../../scenarios/astm/utm/aggregate_checks.md) ([`scenarios.astm.utm.AggregateChecks`](../../../scenarios/astm/utm/aggregate_checks.py)) 6. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Data Validation of GET operational intents by USS](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md) ([`scenarios.astm.utm.data_exchange_validation.GetOpResponseDataValidationByUSS`](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py)) +7. Scenario: [ASTM F3548 UTM aggregate checks](../../../scenarios/astm/utm/aggregate_checks.md) ([`scenarios.astm.utm.AggregateChecks`](../../../scenarios/astm/utm/aggregate_checks.py)) ## [Checked requirements](../../README.md#checked-requirements) @@ -29,7 +30,7 @@