From 2f59e2ff657827a9f8b6b048b4064bb2acea05ef Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 6 Nov 2023 15:39:21 -0800 Subject: [PATCH] [uss_qualifier] Introduce check severity documentation (#319) Introduce check severity documentation --- .../uss_qualifier/common_data_definitions.py | 9 +++++++++ monitoring/uss_qualifier/scenarios/README.md | 9 ++++++++- .../conflict_equal_priority_not_permitted.md | 14 +++++++------- .../validate_not_shared_operational_intent.md | 4 ++-- .../utm/validate_shared_operational_intent.md | 16 ++++++++-------- .../scenarios/documentation/definitions.py | 2 ++ .../scenarios/documentation/parsing.py | 14 +++++++++++++- .../activate_conflict_flight_intent.md | 4 ++-- .../flight_planning/activate_flight_intent.md | 4 ++-- .../modify_activated_conflict_flight_intent.md | 4 ++-- .../modify_planned_conflict_flight_intent.md | 4 ++-- .../plan_conflict_flight_intent.md | 4 ++-- .../flight_planning/plan_flight_intent.md | 4 ++-- 13 files changed, 61 insertions(+), 31 deletions(-) diff --git a/monitoring/uss_qualifier/common_data_definitions.py b/monitoring/uss_qualifier/common_data_definitions.py index 18686c37a2..8983f1594c 100644 --- a/monitoring/uss_qualifier/common_data_definitions.py +++ b/monitoring/uss_qualifier/common_data_definitions.py @@ -85,3 +85,12 @@ def __le__(self, other): else: raise ValueError(f"Cannot compare Severity to {type(other)}") return other >= self + + @property + def symbol(self) -> str: + return { + Severity.Low.value: "ℹī¸", + Severity.Medium.value: "⚠ī¸", + Severity.High.value: "🛑", + Severity.Critical.value: "â˜ĸ", + }.get(self.value, "īŋŊ") diff --git a/monitoring/uss_qualifier/scenarios/README.md b/monitoring/uss_qualifier/scenarios/README.md index 7b15c457f4..fa2d37a7a6 100644 --- a/monitoring/uss_qualifier/scenarios/README.md +++ b/monitoring/uss_qualifier/scenarios/README.md @@ -83,7 +83,14 @@ If the entire test step heading is enclosed in a link, the contents of that link ### Test checks -Each check a test step performs that may result in a finding/issue must be documented via a subsection of the parent test step, named with a " check" suffix (example: `#### Successful injection check`). +Each check a test step performs that may result in a finding/issue must be documented via a subsection of the parent test step, named with a " check" suffix, and a prefix according to the severity of failure of that check (example: `#### 🛑 Successful injection check`). + +The severity of a failure of the check should be indicated with one of the following unicode symbols (these can be copied and pasted into the Markdown documentation): + +* ℹī¸ Low severity: No requirement was violated, but this finding may be useful for improvement. +* ⚠ī¸ Medium severity: A requirement was violated, but the test scenario can continue. +* 🛑 High severity: The test scenario should terminate after cleaning up. +* â˜ĸ Critical severity: Not only can the test scenario not continue, the entire test run should stop. A check should document the requirement(s) violated if the check fails. Requirements are identified by putting a strong emphasis/bold style around the requirement ID (example: `**astm.f3411.v19.NET0420**`). The description of a check should generally explain why the relevant requirement would fail when that information is useful, but the requirement itself should generally not be re-iterated in this description. If the check is self-evident from the requirement, the requirement can be noted without further explanation. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md index aab2519386..5be4e0bec3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md @@ -94,13 +94,13 @@ DSSInstanceResource that provides access to a DSS instance where flight creation ### Check for flight planning readiness test step Both USSs are queried for their readiness to ensure this test can proceed. -#### Flight planning USS not ready check +#### 🛑 Flight planning USS not ready check If either USS does not respond appropriately to the endpoint queried to determine readiness, this check will fail and the USS will have failed to meet **[astm.f3548.v21.GEN0310](../../../../../requirements/astm/f3548/v21.md)** as the USS does not support the InterUSS implementation of that requirement. ### Area clearing test step Both USSs are requested to remove all flights from the area under test. -#### Area cleared successfully check +#### 🛑 Area cleared successfully check **[interuss.automated_testing.flight_planning.ClearArea](../../../../../requirements/interuss/automated_testing/flight_planning.md)** @@ -196,13 +196,13 @@ Do note that executing this test step requires the control USS to support the CM transition to non-conforming state, it will be assumed that the control USS does not support this role and the test execution will stop without failing. -#### Successful transition to non-conforming state check +#### ℹī¸ Successful transition to non-conforming state check All flight intent data provided is correct and notably contains an off-nominal volume, therefore it should have been transitioned to non-conforming state by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. If the USS indicates a conflict, this check will fail. If the USS indicates that the injection attempt failed, this check will fail. -#### Failure check +#### 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. @@ -218,12 +218,12 @@ modification was initiated. While this modification is expected to be accepted b rejection of the modification does not constitute a violation of a requirement. However, the modification request must not result in a failure per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. -#### Successful modification or rejection check +#### 🛑 Successful modification or rejection check All flight intent data provided is correct and the USS should have either successfully modified the flight or rejected properly the modification per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. If the USS indicates that the injection attempt failed, this check will fail. -#### Failure check +#### 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. @@ -235,5 +235,5 @@ either Flight 1 should not have been modified (because the USS kept the original been removed (because the USS rejected the replacement plan provided). ## Cleanup -### Successful flight deletion check +### ⚠ī¸ Successful flight deletion check **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_not_shared_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_not_shared_operational_intent.md index 856e6432bd..fd2e09cdbe 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/validate_not_shared_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_not_shared_operational_intent.md @@ -6,10 +6,10 @@ This assumes an area lock on the extent of the flight intent. See `OpIntentValidator.expect_not_shared()` in [test_steps.py](test_steps.py). -## DSS responses check +## 🛑 DSS responses check **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)** -## Operational intent not shared check +## 🛑 Operational intent not shared check If there are new operational intent references in the area of the flight intent, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../requirements/interuss/automated_testing/flight_planning.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md index b939ffc1ce..8919491cdc 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md @@ -2,36 +2,36 @@ This step verifies that a created flight is shared properly per ASTM F3548-21 by querying the DSS for flights in the area of the flight intent, and then retrieving the details from the USS if the operational intent reference is found. See `OpIntentValidator.expect_shared()` in [test_steps.py](test_steps.py). -## DSS responses check +## 🛑 DSS responses check **[astm.f3548.v21.DSS0005](../../../requirements/astm/f3548/v21.md)** -## Operational intent shared correctly check +## 🛑 Operational intent shared correctly check If a reference to the operational intent for the flight is not found in the DSS, this check will fail per **[astm.f3548.v21.USS0005](../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../requirements/astm/f3548/v21.md)**. -## Operational intent for active flight not deleted check +## 🛑 Operational intent for active flight not deleted check If an activated operational intent is expected to exist after it has been modified or activated and that it is not found in the DSS, this means that there is an active flight without a corresponding operational intent, then this check will fail per **[interuss.automated_testing.flight_planning.FlightCoveredByOperationalIntent](../../../requirements/interuss/automated_testing/flight_planning.md)**. -## Operational intent details retrievable check +## 🛑 Operational intent details retrievable check If the operational intent details for the flight cannot be retrieved from the USS, this check will fail per **[astm.f3548.v21.USS0105](../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../requirements/astm/f3548/v21.md)**. -## Operational intent details data format check +## ⚠ī¸ Operational intent details data format check If the operational intent details response does not validate against [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120), this check fill fail per **[astm.f3548.v21.USS0105](../../../requirements/astm/f3548/v21.md)**. -## Correct operational intent details check +## 🛑 Correct operational intent details check If the operational intent details reported by the USS do not match the user's flight intent, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../requirements/interuss/automated_testing/flight_planning.md)** and **[astm.f3548.v21.OPIN0025](../../../requirements/astm/f3548/v21.md)**. -## Off-nominal volumes check +## ⚠ī¸ Off-nominal volumes check **[astm.f3548.v21.OPIN0015](../../../requirements/astm/f3548/v21.md)** specifies that nominal operational intents (Accepted and Activated) must not include any off-nominal 4D volumes, so this check will fail if an Accepted or Activated operational intent includes off-nominal volumes. -## Vertices check +## ⚠ī¸ Vertices check **[astm.f3548.v21.OPIN0020](../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/documentation/definitions.py b/monitoring/uss_qualifier/scenarios/documentation/definitions.py index ef69179a87..4c36ddb78f 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/definitions.py +++ b/monitoring/uss_qualifier/scenarios/documentation/definitions.py @@ -1,6 +1,7 @@ from typing import Optional, List from implicitdict import ImplicitDict +from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.reports.report import RequirementID @@ -9,6 +10,7 @@ class TestCheckDocumentation(ImplicitDict): url: Optional[str] = None applicable_requirements: List[RequirementID] has_todo: bool + severity: Optional[Severity] = None class TestStepDocumentation(ImplicitDict): diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index c5b27dd3f7..d90ad2cb1e 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -9,6 +9,7 @@ from monitoring import uss_qualifier as uss_qualifier_module from monitoring.monitorlib.inspection import fullname, get_module_object_by_name from monitoring.monitorlib.versioning import repo_url_of +from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.documentation import text_of from monitoring.uss_qualifier.requirements.definitions import RequirementID from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName @@ -58,8 +59,19 @@ def _parse_test_check( reqs.append(RequirementID(text_of(child))) c += 1 + severity = None + for s in Severity: + if name.startswith(s.symbol): + severity = s + name = name[len(s.symbol) :].lstrip() + break + return TestCheckDocumentation( - name=name, url=url, applicable_requirements=reqs, has_todo=has_todo + name=name, + url=url, + applicable_requirements=reqs, + has_todo=has_todo, + severity=severity, ) diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/activate_conflict_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/activate_conflict_flight_intent.md index 86f1dd9f64..1a8aa094d9 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/activate_conflict_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/activate_conflict_flight_intent.md @@ -4,14 +4,14 @@ This page describes the content of a common test step where a user flight intent a non-permitted conflict with an equal priority flight intent. See `activate_conflict_flight_intent` in [prioritization_test_steps.py](prioritization_test_steps.py). -## Incorrectly activated check +## 🛑 Incorrectly activated check If the USS successfully activates the flight or otherwise fails to indicate a conflict, it means they failed to detect the conflict with the pre-existing flight. Therefore, this check will fail if the USS indicates success in creating the flight from the user flight intent, per **[astm.f3548.v21.SCD0045](../../requirements/astm/f3548/v21.md)**. -## Failure check +## 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/activate_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/activate_flight_intent.md index 025ad8c2b5..c61486a0a6 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/activate_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/activate_flight_intent.md @@ -2,11 +2,11 @@ This page describes the content of a common test step where a valid user flight intent should be successfully activated by a flight planner. See `activate_flight_intent` in [test_steps.py](test_steps.py). -## Successful activation check +## 🛑 Successful activation check All flight intent data provided is correct and valid and free of conflict in space and time, therefore it should have been activated by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../requirements/interuss/automated_testing/flight_planning.md)**. If the USS indicates a conflict, this check will fail. If the USS indicates that the flight was rejected, this check will fail. If the USS indicates that the injection attempt failed, this check will fail. -## Failure check +## 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_conflict_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_conflict_flight_intent.md index d9398d176f..9bf08bb9f3 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_conflict_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/modify_activated_conflict_flight_intent.md @@ -4,14 +4,14 @@ This page describes the content of a common test step where a user flight intent of a non-permitted conflict with an equal priority flight intent. See `modify_activated_conflict_flight_intent` in [prioritization_test_steps.py](prioritization_test_steps.py). -## Incorrectly modified check +## 🛑 Incorrectly modified check If the USS successfully modifies the flight or otherwise fails to indicate a conflict, it means they failed to detect the conflict with the pre-existing flight. Therefore, this check will fail if the USS indicates success in modifying the flight from the user flight intent, per **[astm.f3548.v21.SCD0050](../../requirements/astm/f3548/v21.md)**. -## Failure check +## 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/modify_planned_conflict_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/modify_planned_conflict_flight_intent.md index 88e8e2fbd2..bf735663f7 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/modify_planned_conflict_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/modify_planned_conflict_flight_intent.md @@ -4,14 +4,14 @@ This page describes the content of a common test step where a user flight intent of a non-permitted conflict with an equal priority flight intent. See `modify_planned_conflict_flight_intent` in [prioritization_test_steps.py](prioritization_test_steps.py). -## Incorrectly modified check +## 🛑 Incorrectly modified check If the USS successfully modifies the flight or otherwise fails to indicate a conflict, it means they failed to detect the conflict with the pre-existing flight. Therefore, this check will fail if the USS indicates success in modifying the flight from the user flight intent, per **[astm.f3548.v21.SCD0040](../../requirements/astm/f3548/v21.md)**. -## Failure check +## 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/plan_conflict_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/plan_conflict_flight_intent.md index 45bf3e2d08..5820f3f358 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/plan_conflict_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/plan_conflict_flight_intent.md @@ -4,14 +4,14 @@ This page describes the content of a common test step where a user flight intent a non-permitted conflict with an equal priority flight intent. See `plan_conflict_flight_intent` in [prioritization_test_steps.py](prioritization_test_steps.py). -## Incorrectly planned check +## 🛑 Incorrectly planned check If the USS successfully plans the flight or otherwise fails to indicate a conflict, it means they failed to detect the conflict with the pre-existing flight. Therefore, this check will fail if the USS indicates success in creating the flight from the user flight intent, per **[astm.f3548.v21.SCD0035](../../requirements/astm/f3548/v21.md)**. -## Failure check +## 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/plan_flight_intent.md b/monitoring/uss_qualifier/scenarios/flight_planning/plan_flight_intent.md index 58328632bd..ccfbd00d4f 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/plan_flight_intent.md +++ b/monitoring/uss_qualifier/scenarios/flight_planning/plan_flight_intent.md @@ -2,11 +2,11 @@ This page describes the content of a common test case where a valid user flight intent should be successfully planned by a flight planner. See `plan_flight_intent` in [test_steps.py](test_steps.py). -## Successful planning check +## 🛑 Successful planning check All flight intent data provided is correct and valid and free of conflict in space and time, therefore it should have been planned by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../requirements/interuss/automated_testing/flight_planning.md)**. If the USS indicates a conflict, this check will fail. If the USS indicates that the flight was rejected, this check will fail. If the USS indicates that the injection attempt failed, this check will fail. -## Failure check +## 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per