From 9bbecfbd966891f88890129fe458bd55e020b9a5 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 26 Oct 2023 13:50:43 -0700 Subject: [PATCH 01/27] Scenario for testing data validation by SUT for Get operational_intent --- .../scenarios/astm/utm/interuss/__init__.py | 4 +- .../data_exchange_validation/__init__.py | 0 .../scd_get_op_data_validation_by_sut.md | 97 ++++++++ .../test_steps/__init__.py | 0 .../expected_interactions_test_steps.py | 158 ++++++++++++ .../test_steps/invalid_op_test_steps.py | 225 ++++++++++++++++++ .../plan_flight_intent_expect_failed.md | 24 ++ .../validate_get_operational_intent.md | 9 + ...date_no_notification_operational_intent.md | 10 + ...alidate_notification_operational_intent.md | 10 + ...l_intent_but_with_invalid_interuss_data.md | 20 ++ .../flight_intents/non_conflicting.yaml | 106 +++++++++ 12 files changed, 662 insertions(+), 1 deletion(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/__init__.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/__init__.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/expected_interactions_test_steps.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md create mode 100644 monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py index 8b13789179..bd103b711e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py @@ -1 +1,3 @@ - +from .data_exchange_validation.scd_get_op_data_validation_by_sut import ( + ScdGetOpResponseDataValidationBySUT, +) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md new file mode 100644 index 0000000000..eb332e2953 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md @@ -0,0 +1,97 @@ +# SCD Data Validation of GET operational intents by SUT test scenario + +## Description +This test checks that the USS validates correctly the operational intents it creates. +Notably the following requirements: +- **[astm.f3548.v21.OPIN0015](../../../../../requirements/astm/f3548/v21.md)** +- **[astm.f3548.v21.OPIN0020](../../../../../requirements/astm/f3548/v21.md)** +- **[astm.f3548.v21.OPIN0030](../../../../../requirements/astm/f3548/v21.md)** +- **[astm.f3548.v21.OPIN0040](../../../../../requirements/astm/f3548/v21.md)** +- **[astm.f3548.v21.GEN0500](../../../../../requirements/astm/f3548/v21.md)** +- **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** + +## Resources +### flight_intents +FlightIntentsResource that provides the following flight intents: +- `non-conflicting` : + +### control_uss +FlightPlannerResource that will control the data shared for validation testing of sut. + +### sut +FlightPlannerResource that will be tested for its validation of operational intents. + +### dss +DSSInstanceResource that provides access to a DSS instance where flight creation/sharing can be verified. + +### mock_uss +MockUSSResource + +## Setup test case +### Check for flight planning readiness test step +Both USSs are queried for their readiness to ensure this test can proceed. + +#### Flight planning USS not ready check +If either USS does not respond appropriately to the endpoint queried to determine readiness, this check will fail and the USS will have failed to meet **[astm.f3548.v21.GEN0310](../../../../../requirements/astm/f3548/v21.md)** as the USS does not support the InterUSS implementation of that requirement. + +### Area clearing test step +Both USSs are requested to remove all flights from the area under test. + +#### Area cleared successfully check +**[interuss.automated_testing.flight_planning.ClearArea](../../../../../requirements/interuss/automated_testing/flight_planning.md)** + +## Attempt to plan no conflict flight - near existing flight test case +### [Control USS plans flight 2 test step](../../../../flight_planning/plan_flight_intent.md) +Flight 2 on time range B should be successfully planned by the control USS. + +### [Validate flight 2 sharing test step](../../validate_shared_operational_intent.md) +Validate that flight 2 is planned + +### [SUT plans flight 1 test step](../../../../flight_planning/plan_flight_intent.md) +The test driver attempts to plan the flight 1 on time range B via the tested USS. It checks if any conflicts with flight 2 +which is of equal priority and came first. +per **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)**. + +### [Validate flight 1 sharing test step](../../validate_shared_operational_intent.md) +Validate Flight 1 is planned. + +### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) +Flight1 when being planned, SUT needs to GET information of flight 2 + +### [Validate flight1 Notification sent to Control USS test step](test_steps/validate_notification_operational_intent.md) +Flight 2 not notified. As per **[astm.f3548.v21.USS0105](../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** + +### [Delete sut flight test step](../../../../flight_planning/delete_flight_intent.md) +Delete sut flight + +### [Delete control flight test step](../../../../flight_planning/delete_flight_intent.md) +Delete sut flight + +## Attempt to plan deconflicted flight - near a flight sharing invalid data test case +### [Control USS plans flight 2, sharing invalid interuss data test step](../../../../flight_planning/plan_flight_intent.md) +Flight 2 on time range B should be successfully planned by the control USS. + +### [Validate flight 2 shared intent with invalid interuss data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) +Validate that flight 2 is planned + +### [SUT attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) +The test driver attempts to plan the flight 1 on time range B via the tested USS. It checks if any conflicts with flight 2 +which is of equal priority and came first. +per **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)**. + +### [Validate flight 1 not shared by SUT test step](../../validate_not_shared_operational_intent.md) +Validate Flight 1 is not shared. + +### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) +When Flight1 is being planned, SUT needs to GET information of flight 2 + +### [Validate flight1 Notification not sent to Control USS test step](test_steps/validate_no_notification_operational_intent.md) + +### [Delete Control USS flight test step](../../../../flight_planning/delete_flight_intent.md) +Delete control USS flight 2 + +### [Delete SUT flight test step](../../../../flight_planning/delete_flight_intent.md) +Delete control USS flight 2 +## Cleanup +### Successful flight deletion check +**[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/expected_interactions_test_steps.py new file mode 100644 index 0000000000..2879769938 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/expected_interactions_test_steps.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +from typing import List, Optional +import time +import jwt + +from monitoring.monitorlib.fetch import QueryError +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient +from implicitdict import StringBasedDateTime +from loguru import logger +from monitoring.monitorlib.clients.mock_uss.interactions import Interaction + + +def expect_interuss_post_interactions( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + posted_to_url: str, + test_step: str, +): + found = False + if mock_uss is not None: + interactions = _get_interuss_interactions(scenario, mock_uss, st, test_step) + logger.debug(f"Checking for Post to {posted_to_url}") + with scenario.check("Expect Notification sent") as check: + found = False + for interaction in interactions: + method = interaction.query.request.method + url = interaction.query.request.url + if method == "POST" and posted_to_url in url: + found = True + if found == False: + check.record_failed( + summary=f"Notification to {posted_to_url} not received", + severity=Severity.Medium, + details=f"Notification to {posted_to_url} not received", + requirements="SCDxxxx", + ) + scenario.end_test_step() + + return found + + +def expect_no_interuss_post_interactions( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + posted_to_url: str, + test_step: str, +): + found = False + if mock_uss is not None: + interactions = _get_interuss_interactions(scenario, mock_uss, st, test_step) + logger.debug(f"Checking for POST request to {posted_to_url}") + with scenario.check("Expect Notification not sent") as check: + found = False + for interaction in interactions: + method = interaction.query.request.method + url = interaction.query.request.url + if method == "POST" and posted_to_url in url: + found = True + if found == True: + check.record_failed( + summary=f"Notification to {posted_to_url} wrongly sent", + severity=Severity.Medium, + details=f"Notification to {posted_to_url} wrongly sent", + requirements="SCDxxxx", + ) + scenario.end_test_step() + + return found + + +def expect_interuss_get_interactions( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + get_from_url: str, + id: str, + test_step: str, +): + found = False + if mock_uss is not None: + interactions = _get_interuss_interactions(scenario, mock_uss, st, test_step) + logger.debug(f"Checking for GET request to {get_from_url} for id {id}") + with scenario.check("Expect GET request") as check: + found = False + for interaction in interactions: + method = interaction.query.request.method + url = interaction.query.request.url + if method == "GET" and get_from_url in url and id in url: + found = True + if found == False: + check.record_failed( + summary=f"No GET request received at {get_from_url} for {id} ", + severity=Severity.Medium, + details=f"No GET request received at {get_from_url} for {id}", + requirements="SCDxxxx", + ) + + scenario.end_test_step() + return found + + +def _get_interuss_interactions( + scenario: TestScenarioType, + mock_uss: MockUSSClient, + st: StringBasedDateTime, + test_step: str, +) -> List[Interaction]: + scenario.begin_test_step(test_step) + time.sleep(5) + with scenario.check("MockUSS interactions request") as check: + try: + all_interactions, query = mock_uss.get_interactions(st) + scenario.record_query(query) + except QueryError as e: + for q in e.queries: + scenario.record_query(q) + check.record_failed( + summary=f"Error from mock_uss when attempting to get interactions from_time {st}", + severity=Severity.High, + details=f"{str(e)}\n\nStack trace:\n{e.stacktrace}", + query_timestamps=[q.request.timestamp for q in e.queries], + ) + + exclude_sub = mock_uss.session.auth_adapter.get_sub() + + def is_uss_interaction(interaction: Interaction, excl_sub: str) -> bool: + headers = interaction.query.request.headers + if "Authorization" in headers: + token = headers.get("Authorization").split(" ")[1] + payload = jwt.decode( + token, algorithms="RS256", options={"verify_signature": False} + ) + sub = payload["sub"] + logger.debug(f"sub of interuss_interaction token: {sub}") + if sub == excl_sub: + logger.debug(f"Excluding interaction with sub: {sub} ") + return False + else: + return True + else: + logger.error(f"Interaction received without Authorization : {interaction}") + return False + + interuss_interactions = [] + for interaction in all_interactions: + if is_uss_interaction(interaction, exclude_sub): + interuss_interactions.append(interaction) + logger.debug( + f"Interuss interaction reported : {interaction.query.request.method} {interaction.query.request.url} " + f"with response {interaction.query.response.status_code}" + ) + + return interuss_interactions diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py new file mode 100644 index 0000000000..46330c554d --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py @@ -0,0 +1,225 @@ +from typing import Optional +from monitoring.monitorlib import schema_validation, fetch +from implicitdict import ImplicitDict +from uas_standards.astm.f3548.v21.api import ( + OperationalIntentState, + OperationalIntentReference, + GetOperationalIntentDetailsResponse, +) +from loguru import logger +from uas_standards.interuss.automated_testing.scd.v1.api import ( + InjectFlightRequest, + InjectFlightResponseResult, + InjectFlightResponse, +) +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance +from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( + FlightPlanner, +) +from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import OpIntentValidator +from monitoring.uss_qualifier.scenarios.flight_planning.test_steps import ( + submit_flight_intent, + expect_flight_intent_state, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient + + +def plan_flight_intent_expect_failed( + scenario: TestScenarioType, + test_step: str, + flight_planner: FlightPlanner, + flight_intent: InjectFlightRequest, +) -> InjectFlightResponse: + """Attempt to plan a flight intent that would result in a Failed result. + + This function implements the test step described in scd_data_exchange_validation.md. + It validates requirement astm.f3548.v21.SCD00abc. + + Returns: The injection response. + """ + expect_flight_intent_state( + flight_intent, OperationalIntentState.Accepted, scenario, test_step + ) + + return submit_flight_intent( + scenario, + test_step, + "Plan should fail", + {InjectFlightResponseResult.Failed}, + { + InjectFlightResponseResult.Planned: "Failure If Planned", + InjectFlightResponseResult.ConflictWithFlight: "Failure If Conflict", + InjectFlightResponseResult.Rejected: "Failure If Rejected", + }, + flight_planner, + flight_intent, + )[0] + + +class InvalidOpIntentSharingValidator(OpIntentValidator): + def expect_shared_with_invalid_data( + self, flight_intent: InjectFlightRequest, skip_if_not_found: bool = False + ) -> Optional[OperationalIntentReference]: + """Validate that operational intent information was shared with dss for a flight intent, but shared invalid data with USS. + + This function implements the test step described in validate_sharing_operational_intent_but_with_invalid_interuss_data. + + :param flight_intent: the flight intent that was supposed to have been shared. + :param skip_if_not_found: set to True to skip the execution of the checks if the operational intent was not found while it should have been modified. + + :returns: the shared operational intent reference. None if skipped because not found. + """ + self._begin_step() + + with self._scenario.check( + "Operational intent shared with DSS", [self._flight_planner.participant_id] + ) as check: + if self._orig_oi_ref is None: + # we expect a new op intent to have been created + if self._new_oi_ref is None: + check.record_failed( + summary="Operational intent reference not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared a new operational intent with the DSS, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[self._after_query.request.timestamp], + ) + oi_ref = self._new_oi_ref + + elif self._new_oi_ref is None: + # We expect the original op intent to have been either modified or left untouched, thus must be among + # the returned op intents. If additionally the op intent corresponds to an active flight, we fail a + # different appropriate check. Exception made if skip_if_not_found=True and op intent was deleted: step + # is skipped. + modified_oi_ref = self._find_after_oi(self._orig_oi_ref.id) + if modified_oi_ref is None: + if not skip_if_not_found: + if ( + flight_intent.operational_intent.state + == OperationalIntentState.Activated + ): + with self._scenario.check( + "Operational intent for active flight not deleted", + [self._flight_planner.participant_id], + ) as active_flight_check: + active_flight_check.record_failed( + summary="Operational intent reference for active flight not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[ + self._after_query.request.timestamp + ], + ) + else: + check.record_failed( + summary="Operational intent reference not found in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", + query_timestamps=[self._after_query.request.timestamp], + ) + else: + self._scenario.record_note( + self._flight_planner.participant_id, + f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS, instructed to skip test step.", + ) + self._scenario.end_test_step() + return None + oi_ref = modified_oi_ref + + else: + # we expect the original op intent to have been replaced with a new one, thus old one must NOT be among the returned op intents + if self._find_after_oi(self._orig_oi_ref.id) is not None: + check.record_failed( + summary="Operational intent reference found duplicated in DSS", + severity=Severity.High, + details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by replacing it, but it ended up duplicating the operational intent in the DSS", + query_timestamps=[self._after_query.request.timestamp], + ) + oi_ref = self._new_oi_ref + + goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( + oi_ref + ) + self._scenario.record_query(oi_full_query) + with self._scenario.check( + "Operational intent details retrievable", + [self._flight_planner.participant_id], + ) as check: + if oi_full_query.status_code != 200: + check.record_failed( + summary="Operational intent details could not be retrieved from USS", + severity=Severity.High, + details=f"Received status code {oi_full_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {oi_ref.id}", + query_timestamps=[oi_full_query.request.timestamp], + ) + + # if schema validation errors or standard req validation + with self._scenario.check( + "Invalid data in Operational intent details shared by Mock USS for negative test", + [self._flight_planner.participant_id], + ) as check: + + validation_errors = [] + # schema_validation_errors = schema_validation.validate( + # schema_validation.F3548_21.OpenAPIPath, + # schema_validation.F3548_21.GetOperationalIntentDetailsResponse, + # oi_full_query.response.json, + # ) + schema_validation_errors = None + logger.debug(f"Schema validation errors {schema_validation_errors}") + if schema_validation_errors: + details = ( + "The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" + + "\n".join( + f"At {e.json_path} in the response: {e.message}" + for e in schema_validation_errors + ) + ) + validation_errors.append(details) + else: + oi_full = None + try: + goidr = ImplicitDict.parse( + goidr_json, GetOperationalIntentDetailsResponse + ) + oi_full = goidr.operational_intent + + if ( + oi_full.reference.state == OperationalIntentState.Accepted + or oi_full.reference.state == OperationalIntentState.Activated + ) and oi_full.details.get("off_nominal_volumes", None): + details = f"Operational intent {oi_full.reference.id} had {len(oi_full.details.off_nominal_volumes)} off-nominal volumes in wrong state - {oi_full.reference.state}" + validation_errors.append(details) + + def volume_vertices(v4): + if "outline_circle" in v4.volume: + return 1 + if "outline_polygon" in v4.volume: + return len(v4.volume.outline_polygon.vertices) + + all_volumes = oi_full.details.get("volumes", []) + oi_full.details.get( + "off_nominal_volumes", [] + ) + n_vertices = sum(volume_vertices(v) for v in all_volumes) + + if n_vertices > 10000: + details = ( + f"Operational intent {oi_full.reference.id} had too many total vertices - {n_vertices}", + ) + validation_errors.append(details) + except (KeyError, ValueError) as e: + logger.debug(f"Validation error in GetOperationalIntentDetailsResponse. {e}") + validation_errors.append(e) + + if not validation_errors: + check.record_failed( + summary="This negative test case requires invalid data shared with other USS in Operational intent details ", + severity=Severity.High, + details=f"Data shared by Mock USS with other USSes had no invalid data. This test case required invalid data for testing.", + query_timestamps=[oi_full_query.request.timestamp], + ) + + self._scenario.end_test_step() + return oi_ref + diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md new file mode 100644 index 0000000000..03058582e7 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md @@ -0,0 +1,24 @@ +# Plan flight Expect Failed test step + +This page describes the content of a common test case where a valid user flight intent fails in a flight planner, because of invalid data for a nearby flight shared by another USS. See `plan_flight_intent_expect_failed` in [test_steps.py](invalid_op_test_steps.py). + +## Plan should fail check + +Flight intent data of a nearby flight shared was invalid, therefore it should have been failed to plan by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. + +## Failure If Planned check + +Flight intent data of a nearby flight shared was invalid, but the result was planned. It should have been been failed. +If the USS indicates that the injection did not fail, this check will fail per +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. + +## Failure If Conflict check +Flight intent data of a nearby flight shared was invalid, but the result was conflict with flight. It should have been been failed. +If the USS indicates that the injection did not fail, this check will fail per +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. + +## Failure If Rejected check + +Flight intent data of a nearby flight shared was invalid, but the result rejected. It should have been been failed. +If the USS indicates that the injection did not fail, this check will fail per +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md new file mode 100644 index 0000000000..c18b68809f --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md @@ -0,0 +1,9 @@ +# Validate GET interaction test step + +This step verifies that a USS makes a GET request to get the intent_details of an existing operation when needed as per ASTM F3548-21 by checking the interuss interactions of mock uss + +## MockUSS interactions request check +**[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. + +## Expect GET request check +**[astm.f3548.v21.SCD0085](../../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md new file mode 100644 index 0000000000..a6583e220a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md @@ -0,0 +1,10 @@ +# Validate no notification test step + +This step verifies that a created flight is notified properly per ASTM F3548-21 by checking the interuss interactions of mock uss + +## MockUSS interactions request check +**[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. + +## Expect Notification not sent check + +**[astm.f3548.v21.SCD0085](../../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md new file mode 100644 index 0000000000..b32520453a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md @@ -0,0 +1,10 @@ +# Validate notification test step + +This step verifies that a created flight is notified properly per ASTM F3548-21 by checking the interuss interactions of mock uss + +## MockUSS interactions request check +**[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. + +## Expect Notification sent check + +**[astm.f3548.v21.SCD0085](../../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md new file mode 100644 index 0000000000..6cdd35672b --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md @@ -0,0 +1,20 @@ +# Validate flight sharing invalid data test step + +This step verifies that a created flight is shared properly per ASTM F3548-21 by querying the DSS for flights in the area of the flight intent, and then retrieving the details from the USS if the operational intent reference is found. See `validate_shared_operational_intent` in [test_steps.py](invalid_op_test_steps.py). + +## DSS responses check + +**[astm.f3548.v21.DSS0005](../../../../../../requirements/astm/f3548/v21.md)** + +## Operational intent shared with DSS check + +If a reference to the operational intent for the flight is not found in the DSS, this check will fail per **[astm.f3548.v21.USS0005](../../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../../requirements/astm/f3548/v21.md)**. + +## Operational intent details retrievable check + +If the operational intent details for the flight cannot be retrieved from the USS, this check will fail per **[astm.f3548.v21.USS0105](../../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../../requirements/astm/f3548/v21.md)**. + +## Invalid data in Operational intent details shared by Mock USS for negative test check + +Mock USS shares operational intent details response for the negative test case as per [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120) but the data is invalid, this check fill fail per **[astm.f3548.v21.USS0105](../../../../../../requirements/astm/f3548/v21.md)**. + diff --git a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml new file mode 100644 index 0000000000..af7d58d10e --- /dev/null +++ b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml @@ -0,0 +1,106 @@ +intents: + flight_1_planned_time_range_A: + full: + basic_information: + usage_state: Planned + uas_state: Nominal + area: + - outline_polygon: + vertices: + - lat: 37.20642344604623 + lng: -80.58508994131496 + - lat: 37.21359527809636 + lng: -80.5767365824006 + - lat: 37.215812746083586 + lng: -80.58144645497978 + - lat: 37.2146332499489 + lng: -80.58408279875103 + altitude_lower: + value: 474 + reference: W84 + units: M + altitude_upper: + value: 560 + reference: W84 + units: M + start_time: + offset_from: + starting_from: + start_of_test: { } + offset: -1s + duration: 5m + + astm_f3548_21: + priority: 0 + + uspace_flight_authorisation: + uas_serial_number: 1AF49UL5CC5J6K + operation_category: Open + operation_mode: Vlos + uas_class: C0 + identification_technologies: + - ASTMNetRID + connectivity_methods: + - cellular + endurance_minutes: 30 + emergency_procedure_url: https://example.interussplatform.org/emergency + operator_id: CHEo5kut30e0mt01-qwe + uas_id: '' + uas_type_certificate: '' + + + flight_2_planned_time_range_A: + full: + basic_information: + usage_state: Planned + uas_state: Nominal + area: + - outline_polygon: + vertices: + - lat: 37.214119359044275 + lng: -80.58443600701524 + - lat: 37.212388260776436 + lng: -80.58824885811198 + - lat: 37.20211436858821 + lng: -80.5856832012991 + - lat: 37.21394908884487 + lng: -80.57474352572189 + - lat: 37.206173006943104 + lng: -80.5852199577081 + altitude_lower: + value: 483 + reference: W84 + units: M + altitude_upper: + value: 519 + reference: W84 + units: M + start_time: + offset_from: + starting_from: + start_of_test: { } + offset: -1s + duration: 5m + + astm_f3548_21: + priority: 0 # TODO: Remove flight_authorisation section when it is optional + + uspace_flight_authorisation: + uas_serial_number: 1AF49UL5CC5J6K + operation_category: Open + operation_mode: Vlos + uas_class: C0 + identification_technologies: + - ASTMNetRID + connectivity_methods: + - cellular + endurance_minutes: 30 + emergency_procedure_url: https://example.interussplatform.org/emergency + operator_id: CHEo5kut30e0mt01-qwe + uas_id: '' + uas_type_certificate: '' + + + + + From 1fd83d580f75ce77161bad05e6ffb12161a309e3 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 26 Oct 2023 14:01:24 -0700 Subject: [PATCH 02/27] Removing an import --- .../uss_qualifier/scenarios/astm/utm/interuss/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py index bd103b711e..8b13789179 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/__init__.py @@ -1,3 +1 @@ -from .data_exchange_validation.scd_get_op_data_validation_by_sut import ( - ScdGetOpResponseDataValidationBySUT, -) + From 5d618a2d468d64a96037a0476e4a361bf2ccd2f7 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 26 Oct 2023 15:14:26 -0700 Subject: [PATCH 03/27] Fix format --- .../test_steps/invalid_op_test_steps.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py index 46330c554d..7969f07e3f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py @@ -198,9 +198,9 @@ def volume_vertices(v4): if "outline_polygon" in v4.volume: return len(v4.volume.outline_polygon.vertices) - all_volumes = oi_full.details.get("volumes", []) + oi_full.details.get( - "off_nominal_volumes", [] - ) + all_volumes = oi_full.details.get( + "volumes", [] + ) + oi_full.details.get("off_nominal_volumes", []) n_vertices = sum(volume_vertices(v) for v in all_volumes) if n_vertices > 10000: @@ -209,7 +209,9 @@ def volume_vertices(v4): ) validation_errors.append(details) except (KeyError, ValueError) as e: - logger.debug(f"Validation error in GetOperationalIntentDetailsResponse. {e}") + logger.debug( + f"Validation error in GetOperationalIntentDetailsResponse. {e}" + ) validation_errors.append(e) if not validation_errors: @@ -222,4 +224,3 @@ def volume_vertices(v4): self._scenario.end_test_step() return oi_ref - From 56e0e8b805aa4d6e9159e10243e7bf22ab094c08 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 26 Oct 2023 15:41:15 -0700 Subject: [PATCH 04/27] Add interactions requirement file --- .../requirements/interuss/mock_uss/interactions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md diff --git a/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md b/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md new file mode 100644 index 0000000000..fb2cfed270 --- /dev/null +++ b/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md @@ -0,0 +1,10 @@ +InterUss mock_uss interactions requirements + +## Overview +InterUSS provides endpoint to get InterUUS interactions recorded by mock_uss + + +## Requirements + +### Interactions +GET interactions from a time instance to now From 427cdf7fefa7d363fd24a312e3141ee3dfc0dbde Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 26 Oct 2023 16:38:37 -0700 Subject: [PATCH 05/27] Adding more description to the scenario --- .../scd_get_op_data_validation_by_sut.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md index eb332e2953..b7e5a95a5d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md @@ -1,7 +1,9 @@ # SCD Data Validation of GET operational intents by SUT test scenario ## Description -This test checks that the USS validates correctly the operational intents it creates. +This test checks that the SUT validates the operational intents received as response to its GET request from another USS. +Control uss which is a mock uss plans a nearby V-shaped operation, and provides the data that SUT GETs. +SUT validates the GET response to successfully plan its operation Notably the following requirements: - **[astm.f3548.v21.OPIN0015](../../../../../requirements/astm/f3548/v21.md)** - **[astm.f3548.v21.OPIN0020](../../../../../requirements/astm/f3548/v21.md)** @@ -12,8 +14,8 @@ Notably the following requirements: ## Resources ### flight_intents -FlightIntentsResource that provides the following flight intents: -- `non-conflicting` : +FlightIntentsResource that provides the following V-shaped flight intents: +- `non-conflicting` : flight1 and flight2 ### control_uss FlightPlannerResource that will control the data shared for validation testing of sut. From 2dddc76e82b262be61469b480aa9fb37b75d5b4f Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 26 Oct 2023 22:18:41 -0700 Subject: [PATCH 06/27] Scenario documentation fix --- .../scd_get_op_data_validation_by_sut.md | 1 + .../test_steps/validate_get_operational_intent.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md index b7e5a95a5d..159528892b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md @@ -94,6 +94,7 @@ Delete control USS flight 2 ### [Delete SUT flight test step](../../../../flight_planning/delete_flight_intent.md) Delete control USS flight 2 + ## Cleanup ### Successful flight deletion check **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md index c18b68809f..ae0f3de90d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md @@ -6,4 +6,4 @@ This step verifies that a USS makes a GET request to get the intent_details of a **[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. ## Expect GET request check -**[astm.f3548.v21.SCD0085](../../../../../../requirements/astm/f3548/v21.md)** +**[astm.f3548.v21.SCD0075](../../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.SCD0035](../../../../../../requirements/astm/f3548/v21.md)** From db471a081de62a7a8bd60612dc284df6890f9a72 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Fri, 27 Oct 2023 14:57:44 -0700 Subject: [PATCH 07/27] Improved requirements description --- .../requirements/interuss/mock_uss/interactions.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md b/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md index fb2cfed270..f46ca6351c 100644 --- a/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md +++ b/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md @@ -1,10 +1,9 @@ InterUss mock_uss interactions requirements ## Overview -InterUSS provides endpoint to get InterUUS interactions recorded by mock_uss - +Mock_uss records interactions on its uss/v1/operational_intents endpoints. It provides an endpoint for uss_qualifier to get these interactions for testing various scenarios. ## Requirements ### Interactions -GET interactions from a time instance to now +GET /mock_uss/interuss_logging/logs returns a list of interactions that took place between the time specified by query parameter from_time and the current time. From fdc3c7fa158ffac870aedd4822431b45a8f241f2 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Tue, 31 Oct 2023 00:00:14 -0700 Subject: [PATCH 08/27] Per PR comments, fixed module path, file name, checks and description in the test scenario and test steps readme files --- .../data_exchange_validation/__init__.py | 0 .../get_op_data_validation.md | 108 ++++++++++++++++++ .../test_steps/__init__.py | 0 .../expected_interactions_test_steps.py | 0 .../test_steps/invalid_op_test_steps.py | 0 .../plan_flight_intent_expect_failed.md | 8 +- .../validate_get_operational_intent.md | 10 ++ ...date_no_notification_operational_intent.md | 10 ++ ...alidate_notification_operational_intent.md | 12 ++ ...l_intent_but_with_invalid_interuss_data.md | 8 +- .../scd_get_op_data_validation_by_sut.md | 100 ---------------- .../validate_get_operational_intent.md | 9 -- ...date_no_notification_operational_intent.md | 10 -- ...alidate_notification_operational_intent.md | 10 -- 14 files changed, 148 insertions(+), 137 deletions(-) rename monitoring/uss_qualifier/scenarios/astm/utm/{interuss => }/data_exchange_validation/__init__.py (100%) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md rename monitoring/uss_qualifier/scenarios/astm/utm/{interuss => }/data_exchange_validation/test_steps/__init__.py (100%) rename monitoring/uss_qualifier/scenarios/astm/utm/{interuss => }/data_exchange_validation/test_steps/expected_interactions_test_steps.py (100%) rename monitoring/uss_qualifier/scenarios/astm/utm/{interuss => }/data_exchange_validation/test_steps/invalid_op_test_steps.py (100%) rename monitoring/uss_qualifier/scenarios/astm/utm/{interuss => }/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md (81%) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md rename monitoring/uss_qualifier/scenarios/astm/utm/{interuss => }/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md (74%) delete mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md delete mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md delete mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md delete mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/__init__.py similarity index 100% rename from monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/__init__.py rename to monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/__init__.py diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md new file mode 100644 index 0000000000..5721888ddb --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -0,0 +1,108 @@ +# SCD Data Validation of GET operational intents by USS test scenario + +## Description +This test checks that the USS being tested validates the operational intents received as response to its GET request from another USS. +Control_uss which is a mock uss plans a nearby V-shaped operation, and provides the data that tested_uss GETs. +tested_uss validates the GET response from control_uss and accordingly plan its operation. +Notably the following requirements: + +- **[astm.f3548.v21.SCD0035](../../../../requirements/astm/f3548/v21.md)** + +## Resources +### flight_intents +FlightIntentsResource provides the two V-shaped flight intents. +The convex hulls of the 2D footprints of the two flights intersect, but the polygons do not intersect. +There is an overlap in time and altitude of the two flights. +- flight_1_planned_time_range_A +- flight_2_planned_time_range_A + +### control_uss +MockUSSResource that will be used for planning flights, controlling data shared for validation testing, and gathering interuss interactions from mock_uss. + +### tested_uss +FlightPlannerResource that will be used for the USS being tested for its data validation of operational intent. + +### dss +DSSInstanceResource that provides access to a DSS instance where flight creation/sharing can be verified. + +## Setup test case +### Check for flight planning readiness test step +Both USSs are queried for their readiness to ensure this test can proceed. + +#### Flight planning USS not ready check +If either USS does not respond appropriately to the endpoint queried to determine readiness, this check will fail and the USS will have failed to meet **[astm.f3548.v21.GEN0310](../../../../requirements/astm/f3548/v21.md)** as the USS does not support the InterUSS implementation of that requirement. + +### Area clearing test step +Both USSs are requested to remove all flights from the area under test. + +#### Area cleared successfully check +**[interuss.automated_testing.flight_planning.ClearArea](../../../../requirements/interuss/automated_testing/flight_planning.md)** + +## Successfully plan flight near an existing flight test case +### [Control_uss plans flight 2 test step](../../../flight_planning/plan_flight_intent.md) +Flight 2 on time range A should be successfully planned by the control USS. + +### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) +Validate that flight 2 is planned + +### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) +There should be no subscription by tested_uss to trigger notification of flight 2. +This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. +If this notification was pushed, the GET operational intent data validation test cannot be done. + +### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) +The test driver attempts to plan flight 1 on time range A via the tested USS. It checks if any conflicts with flight 2 +which is of equal priority and came first. +per **[astm.f3548.v21.SCD0035](../../../../requirements/astm/f3548/v21.md)**. + +### [Validate flight 1 sharing test step](../validate_shared_operational_intent.md) +Validate flight 1 is planned. + +### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) +Tested_uss needs to make GET request for obtaining details of flight 2. +In a previous step, we checked there was no notification of flight 2 to tested_uss. + +### [Validate flight1 Notification sent to Control_uss test step](test_steps/validate_notification_operational_intent.md) +Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. As per **[astm.f3548.v21.USS0105](../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.SCD0085](../../../../requirements/astm/f3548/v21.md)** + +### [Delete tested_uss flight test step](../../../flight_planning/delete_flight_intent.md) +Teardown + +### [Delete control_uss flight test step](../../../flight_planning/delete_flight_intent.md) +Teardown + +## Flight planning prevented due to invalid data sharing test case +### [Control_uss plans flight 2, sharing invalid interuss data test step](../../../flight_planning/plan_flight_intent.md) +Flight 2 on time range A should be successfully planned by the control USS. + +### [Validate flight 2 shared intent with invalid interuss data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) +Validate that flight 2 is planned + +### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) +There should be no subscription by tested_uss to trigger notification of flight 2. +This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. +If this notification was pushed, the GET operational intent data validation test cannot be done. + +### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) +The test driver attempts to plan the flight 1 on time range A via the tested_uss. It checks if any conflicts with flight 2 +which is of equal priority and came first. +per **[astm.f3548.v21.SCD0035](../../../../requirements/astm/f3548/v21.md)**. + +### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) +Validate flight 1 is not shared. + +### [Validate flight 2 GET interaction test step](test_steps/validate_get_operational_intent.md) +Tested_uss needs to make GET request for obtaining details of flight 2 from control_uss. +In a previous step, we checked there was no notification of flight 2 to tested_uss. + +### [Validate flight 1 Notification not sent to Control_uss test step](test_steps/validate_no_notification_operational_intent.md) + +### [Delete Control_uss flight test step](../../../flight_planning/delete_flight_intent.md) +Teardown + +### [Delete tested_uss flight test step](../../../flight_planning/delete_flight_intent.md) +Teardown if created. + +## Cleanup +### Successful flight deletion check +**[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/__init__.py similarity index 100% rename from monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/__init__.py rename to monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/__init__.py diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py similarity index 100% rename from monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/expected_interactions_test_steps.py rename to monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py similarity index 100% rename from monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/invalid_op_test_steps.py rename to monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md similarity index 81% rename from monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md rename to monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md index 03058582e7..6581b593da 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md @@ -4,21 +4,21 @@ This page describes the content of a common test case where a valid user flight ## Plan should fail check -Flight intent data of a nearby flight shared was invalid, therefore it should have been failed to plan by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +Flight intent data of a nearby flight shared was invalid, therefore it should have been failed to plan by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. ## Failure If Planned check Flight intent data of a nearby flight shared was invalid, but the result was planned. It should have been been failed. If the USS indicates that the injection did not fail, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. ## Failure If Conflict check Flight intent data of a nearby flight shared was invalid, but the result was conflict with flight. It should have been been failed. If the USS indicates that the injection did not fail, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. ## Failure If Rejected check Flight intent data of a nearby flight shared was invalid, but the result rejected. It should have been been failed. If the USS indicates that the injection did not fail, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md new file mode 100644 index 0000000000..bd97d824e3 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md @@ -0,0 +1,10 @@ +# Validate GET interaction test step + +This step verifies that a USS makes a GET request to get the intent_details of an existing operation when needed as per ASTM F3548-21 by checking the interuss interactions of mock uss + +## MockUSS interactions request check +**[interuss.mock_uss.interactions.Interactions](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + +## Expect GET request check +**[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** +SCD0035 needs a USS to verify before transitioning to Accepted that it does not conflict with a type of operational intent, and the only way to have verified this is by knowing all operational intent details, and (from previous checks of no notifications) the only way to know the operational intent details of flight is to have requested them via a GET details interaction. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md new file mode 100644 index 0000000000..e534f77f39 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md @@ -0,0 +1,10 @@ +# Validate no notification test step + +This step verifies when a flight is not created, it is also not notified by checking the interuss interactions of mock_uss instance. + +## MockUSS interactions request check +**[interuss.mock_uss.interactions.Interactions](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + +## Expect Notification not sent check + +**[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md new file mode 100644 index 0000000000..26ff33ec32 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md @@ -0,0 +1,12 @@ +# Validate notification test step + +This step verifies that, when creating or modifying an operational intent, a USS sent the required notification for a relevant subscription owned by a mock_uss instance by checking the interactions of that mock_uss instance. + +## MockUSS interactions request check +**[interuss.mock_uss.interactions.Interactions](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + +## Expect Notification sent check +**[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** + +## Notification data is valid check +**[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md similarity index 74% rename from monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md rename to monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md index 6cdd35672b..e13318769e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md @@ -4,17 +4,17 @@ This step verifies that a created flight is shared properly per ASTM F3548-21 by ## DSS responses check -**[astm.f3548.v21.DSS0005](../../../../../../requirements/astm/f3548/v21.md)** +**[astm.f3548.v21.DSS0005](../../../../../requirements/astm/f3548/v21.md)** ## Operational intent shared with DSS check -If a reference to the operational intent for the flight is not found in the DSS, this check will fail per **[astm.f3548.v21.USS0005](../../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../../requirements/astm/f3548/v21.md)**. +If a reference to the operational intent for the flight is not found in the DSS, this check will fail per **[astm.f3548.v21.USS0005](../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../requirements/astm/f3548/v21.md)**. ## Operational intent details retrievable check -If the operational intent details for the flight cannot be retrieved from the USS, this check will fail per **[astm.f3548.v21.USS0105](../../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../../requirements/astm/f3548/v21.md)**. +If the operational intent details for the flight cannot be retrieved from the USS, this check will fail per **[astm.f3548.v21.USS0105](../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.OPIN0025](../../../../../requirements/astm/f3548/v21.md)**. ## Invalid data in Operational intent details shared by Mock USS for negative test check -Mock USS shares operational intent details response for the negative test case as per [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120) but the data is invalid, this check fill fail per **[astm.f3548.v21.USS0105](../../../../../../requirements/astm/f3548/v21.md)**. +Mock USS shares operational intent details response for the negative test case as per [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120) but the data is invalid, this check fill fail per **[astm.f3548.v21.USS0105](../../../../../requirements/astm/f3548/v21.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md deleted file mode 100644 index 159528892b..0000000000 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/scd_get_op_data_validation_by_sut.md +++ /dev/null @@ -1,100 +0,0 @@ -# SCD Data Validation of GET operational intents by SUT test scenario - -## Description -This test checks that the SUT validates the operational intents received as response to its GET request from another USS. -Control uss which is a mock uss plans a nearby V-shaped operation, and provides the data that SUT GETs. -SUT validates the GET response to successfully plan its operation -Notably the following requirements: -- **[astm.f3548.v21.OPIN0015](../../../../../requirements/astm/f3548/v21.md)** -- **[astm.f3548.v21.OPIN0020](../../../../../requirements/astm/f3548/v21.md)** -- **[astm.f3548.v21.OPIN0030](../../../../../requirements/astm/f3548/v21.md)** -- **[astm.f3548.v21.OPIN0040](../../../../../requirements/astm/f3548/v21.md)** -- **[astm.f3548.v21.GEN0500](../../../../../requirements/astm/f3548/v21.md)** -- **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** - -## Resources -### flight_intents -FlightIntentsResource that provides the following V-shaped flight intents: -- `non-conflicting` : flight1 and flight2 - -### control_uss -FlightPlannerResource that will control the data shared for validation testing of sut. - -### sut -FlightPlannerResource that will be tested for its validation of operational intents. - -### dss -DSSInstanceResource that provides access to a DSS instance where flight creation/sharing can be verified. - -### mock_uss -MockUSSResource - -## Setup test case -### Check for flight planning readiness test step -Both USSs are queried for their readiness to ensure this test can proceed. - -#### Flight planning USS not ready check -If either USS does not respond appropriately to the endpoint queried to determine readiness, this check will fail and the USS will have failed to meet **[astm.f3548.v21.GEN0310](../../../../../requirements/astm/f3548/v21.md)** as the USS does not support the InterUSS implementation of that requirement. - -### Area clearing test step -Both USSs are requested to remove all flights from the area under test. - -#### Area cleared successfully check -**[interuss.automated_testing.flight_planning.ClearArea](../../../../../requirements/interuss/automated_testing/flight_planning.md)** - -## Attempt to plan no conflict flight - near existing flight test case -### [Control USS plans flight 2 test step](../../../../flight_planning/plan_flight_intent.md) -Flight 2 on time range B should be successfully planned by the control USS. - -### [Validate flight 2 sharing test step](../../validate_shared_operational_intent.md) -Validate that flight 2 is planned - -### [SUT plans flight 1 test step](../../../../flight_planning/plan_flight_intent.md) -The test driver attempts to plan the flight 1 on time range B via the tested USS. It checks if any conflicts with flight 2 -which is of equal priority and came first. -per **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)**. - -### [Validate flight 1 sharing test step](../../validate_shared_operational_intent.md) -Validate Flight 1 is planned. - -### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) -Flight1 when being planned, SUT needs to GET information of flight 2 - -### [Validate flight1 Notification sent to Control USS test step](test_steps/validate_notification_operational_intent.md) -Flight 2 not notified. As per **[astm.f3548.v21.USS0105](../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** - -### [Delete sut flight test step](../../../../flight_planning/delete_flight_intent.md) -Delete sut flight - -### [Delete control flight test step](../../../../flight_planning/delete_flight_intent.md) -Delete sut flight - -## Attempt to plan deconflicted flight - near a flight sharing invalid data test case -### [Control USS plans flight 2, sharing invalid interuss data test step](../../../../flight_planning/plan_flight_intent.md) -Flight 2 on time range B should be successfully planned by the control USS. - -### [Validate flight 2 shared intent with invalid interuss data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) -Validate that flight 2 is planned - -### [SUT attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) -The test driver attempts to plan the flight 1 on time range B via the tested USS. It checks if any conflicts with flight 2 -which is of equal priority and came first. -per **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)**. - -### [Validate flight 1 not shared by SUT test step](../../validate_not_shared_operational_intent.md) -Validate Flight 1 is not shared. - -### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) -When Flight1 is being planned, SUT needs to GET information of flight 2 - -### [Validate flight1 Notification not sent to Control USS test step](test_steps/validate_no_notification_operational_intent.md) - -### [Delete Control USS flight test step](../../../../flight_planning/delete_flight_intent.md) -Delete control USS flight 2 - -### [Delete SUT flight test step](../../../../flight_planning/delete_flight_intent.md) -Delete control USS flight 2 - -## Cleanup -### Successful flight deletion check -**[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md deleted file mode 100644 index ae0f3de90d..0000000000 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_get_operational_intent.md +++ /dev/null @@ -1,9 +0,0 @@ -# Validate GET interaction test step - -This step verifies that a USS makes a GET request to get the intent_details of an existing operation when needed as per ASTM F3548-21 by checking the interuss interactions of mock uss - -## MockUSS interactions request check -**[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. - -## Expect GET request check -**[astm.f3548.v21.SCD0075](../../../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.SCD0035](../../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md deleted file mode 100644 index a6583e220a..0000000000 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md +++ /dev/null @@ -1,10 +0,0 @@ -# Validate no notification test step - -This step verifies that a created flight is notified properly per ASTM F3548-21 by checking the interuss interactions of mock uss - -## MockUSS interactions request check -**[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. - -## Expect Notification not sent check - -**[astm.f3548.v21.SCD0085](../../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md deleted file mode 100644 index b32520453a..0000000000 --- a/monitoring/uss_qualifier/scenarios/astm/utm/interuss/data_exchange_validation/test_steps/validate_notification_operational_intent.md +++ /dev/null @@ -1,10 +0,0 @@ -# Validate notification test step - -This step verifies that a created flight is notified properly per ASTM F3548-21 by checking the interuss interactions of mock uss - -## MockUSS interactions request check -**[interuss.mock_uss.interactions.Interactions](../../../../../../requirements/interuss/mock_uss/interactions.md)**. - -## Expect Notification sent check - -**[astm.f3548.v21.SCD0085](../../../../../../requirements/astm/f3548/v21.md)** From 572bfeb21fed5b1ed571d7a1d1349f9947715618 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Tue, 31 Oct 2023 11:30:29 -0700 Subject: [PATCH 09/27] Per PR comments removing py files, and improving description --- .../get_op_data_validation.md | 6 +- .../expected_interactions_test_steps.py | 158 ------------ .../test_steps/invalid_op_test_steps.py | 226 ------------------ .../plan_flight_intent_expect_failed.md | 2 +- ...l_intent_but_with_invalid_interuss_data.md | 2 +- 5 files changed, 4 insertions(+), 390 deletions(-) delete mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py delete mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 5721888ddb..d3371833f0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -76,7 +76,7 @@ Teardown Flight 2 on time range A should be successfully planned by the control USS. ### [Validate flight 2 shared intent with invalid interuss data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) -Validate that flight 2 is planned +Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. ### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) There should be no subscription by tested_uss to trigger notification of flight 2. @@ -98,11 +98,9 @@ In a previous step, we checked there was no notification of flight 2 to tested_u ### [Validate flight 1 Notification not sent to Control_uss test step](test_steps/validate_no_notification_operational_intent.md) ### [Delete Control_uss flight test step](../../../flight_planning/delete_flight_intent.md) -Teardown -### [Delete tested_uss flight test step](../../../flight_planning/delete_flight_intent.md) -Teardown if created. ## Cleanup ### Successful flight deletion check +This cleanup is for both - after testcase ends and after test scenario ends **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py deleted file mode 100644 index 2879769938..0000000000 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/expected_interactions_test_steps.py +++ /dev/null @@ -1,158 +0,0 @@ -from __future__ import annotations - -from typing import List, Optional -import time -import jwt - -from monitoring.monitorlib.fetch import QueryError -from monitoring.uss_qualifier.common_data_definitions import Severity -from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType -from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient -from implicitdict import StringBasedDateTime -from loguru import logger -from monitoring.monitorlib.clients.mock_uss.interactions import Interaction - - -def expect_interuss_post_interactions( - scenario: TestScenarioType, - mock_uss: MockUSSClient, - st: StringBasedDateTime, - posted_to_url: str, - test_step: str, -): - found = False - if mock_uss is not None: - interactions = _get_interuss_interactions(scenario, mock_uss, st, test_step) - logger.debug(f"Checking for Post to {posted_to_url}") - with scenario.check("Expect Notification sent") as check: - found = False - for interaction in interactions: - method = interaction.query.request.method - url = interaction.query.request.url - if method == "POST" and posted_to_url in url: - found = True - if found == False: - check.record_failed( - summary=f"Notification to {posted_to_url} not received", - severity=Severity.Medium, - details=f"Notification to {posted_to_url} not received", - requirements="SCDxxxx", - ) - scenario.end_test_step() - - return found - - -def expect_no_interuss_post_interactions( - scenario: TestScenarioType, - mock_uss: MockUSSClient, - st: StringBasedDateTime, - posted_to_url: str, - test_step: str, -): - found = False - if mock_uss is not None: - interactions = _get_interuss_interactions(scenario, mock_uss, st, test_step) - logger.debug(f"Checking for POST request to {posted_to_url}") - with scenario.check("Expect Notification not sent") as check: - found = False - for interaction in interactions: - method = interaction.query.request.method - url = interaction.query.request.url - if method == "POST" and posted_to_url in url: - found = True - if found == True: - check.record_failed( - summary=f"Notification to {posted_to_url} wrongly sent", - severity=Severity.Medium, - details=f"Notification to {posted_to_url} wrongly sent", - requirements="SCDxxxx", - ) - scenario.end_test_step() - - return found - - -def expect_interuss_get_interactions( - scenario: TestScenarioType, - mock_uss: MockUSSClient, - st: StringBasedDateTime, - get_from_url: str, - id: str, - test_step: str, -): - found = False - if mock_uss is not None: - interactions = _get_interuss_interactions(scenario, mock_uss, st, test_step) - logger.debug(f"Checking for GET request to {get_from_url} for id {id}") - with scenario.check("Expect GET request") as check: - found = False - for interaction in interactions: - method = interaction.query.request.method - url = interaction.query.request.url - if method == "GET" and get_from_url in url and id in url: - found = True - if found == False: - check.record_failed( - summary=f"No GET request received at {get_from_url} for {id} ", - severity=Severity.Medium, - details=f"No GET request received at {get_from_url} for {id}", - requirements="SCDxxxx", - ) - - scenario.end_test_step() - return found - - -def _get_interuss_interactions( - scenario: TestScenarioType, - mock_uss: MockUSSClient, - st: StringBasedDateTime, - test_step: str, -) -> List[Interaction]: - scenario.begin_test_step(test_step) - time.sleep(5) - with scenario.check("MockUSS interactions request") as check: - try: - all_interactions, query = mock_uss.get_interactions(st) - scenario.record_query(query) - except QueryError as e: - for q in e.queries: - scenario.record_query(q) - check.record_failed( - summary=f"Error from mock_uss when attempting to get interactions from_time {st}", - severity=Severity.High, - details=f"{str(e)}\n\nStack trace:\n{e.stacktrace}", - query_timestamps=[q.request.timestamp for q in e.queries], - ) - - exclude_sub = mock_uss.session.auth_adapter.get_sub() - - def is_uss_interaction(interaction: Interaction, excl_sub: str) -> bool: - headers = interaction.query.request.headers - if "Authorization" in headers: - token = headers.get("Authorization").split(" ")[1] - payload = jwt.decode( - token, algorithms="RS256", options={"verify_signature": False} - ) - sub = payload["sub"] - logger.debug(f"sub of interuss_interaction token: {sub}") - if sub == excl_sub: - logger.debug(f"Excluding interaction with sub: {sub} ") - return False - else: - return True - else: - logger.error(f"Interaction received without Authorization : {interaction}") - return False - - interuss_interactions = [] - for interaction in all_interactions: - if is_uss_interaction(interaction, exclude_sub): - interuss_interactions.append(interaction) - logger.debug( - f"Interuss interaction reported : {interaction.query.request.method} {interaction.query.request.url} " - f"with response {interaction.query.response.status_code}" - ) - - return interuss_interactions diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py deleted file mode 100644 index 7969f07e3f..0000000000 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/invalid_op_test_steps.py +++ /dev/null @@ -1,226 +0,0 @@ -from typing import Optional -from monitoring.monitorlib import schema_validation, fetch -from implicitdict import ImplicitDict -from uas_standards.astm.f3548.v21.api import ( - OperationalIntentState, - OperationalIntentReference, - GetOperationalIntentDetailsResponse, -) -from loguru import logger -from uas_standards.interuss.automated_testing.scd.v1.api import ( - InjectFlightRequest, - InjectFlightResponseResult, - InjectFlightResponse, -) -from monitoring.uss_qualifier.common_data_definitions import Severity -from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance -from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( - FlightPlanner, -) -from monitoring.uss_qualifier.scenarios.astm.utm.test_steps import OpIntentValidator -from monitoring.uss_qualifier.scenarios.flight_planning.test_steps import ( - submit_flight_intent, - expect_flight_intent_state, -) -from monitoring.uss_qualifier.scenarios.scenario import TestScenarioType -from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient - - -def plan_flight_intent_expect_failed( - scenario: TestScenarioType, - test_step: str, - flight_planner: FlightPlanner, - flight_intent: InjectFlightRequest, -) -> InjectFlightResponse: - """Attempt to plan a flight intent that would result in a Failed result. - - This function implements the test step described in scd_data_exchange_validation.md. - It validates requirement astm.f3548.v21.SCD00abc. - - Returns: The injection response. - """ - expect_flight_intent_state( - flight_intent, OperationalIntentState.Accepted, scenario, test_step - ) - - return submit_flight_intent( - scenario, - test_step, - "Plan should fail", - {InjectFlightResponseResult.Failed}, - { - InjectFlightResponseResult.Planned: "Failure If Planned", - InjectFlightResponseResult.ConflictWithFlight: "Failure If Conflict", - InjectFlightResponseResult.Rejected: "Failure If Rejected", - }, - flight_planner, - flight_intent, - )[0] - - -class InvalidOpIntentSharingValidator(OpIntentValidator): - def expect_shared_with_invalid_data( - self, flight_intent: InjectFlightRequest, skip_if_not_found: bool = False - ) -> Optional[OperationalIntentReference]: - """Validate that operational intent information was shared with dss for a flight intent, but shared invalid data with USS. - - This function implements the test step described in validate_sharing_operational_intent_but_with_invalid_interuss_data. - - :param flight_intent: the flight intent that was supposed to have been shared. - :param skip_if_not_found: set to True to skip the execution of the checks if the operational intent was not found while it should have been modified. - - :returns: the shared operational intent reference. None if skipped because not found. - """ - self._begin_step() - - with self._scenario.check( - "Operational intent shared with DSS", [self._flight_planner.participant_id] - ) as check: - if self._orig_oi_ref is None: - # we expect a new op intent to have been created - if self._new_oi_ref is None: - check.record_failed( - summary="Operational intent reference not found in DSS", - severity=Severity.High, - details=f"USS {self._flight_planner.participant_id} was supposed to have shared a new operational intent with the DSS, but no matching operational intent references were found in the DSS in the area of the flight intent", - query_timestamps=[self._after_query.request.timestamp], - ) - oi_ref = self._new_oi_ref - - elif self._new_oi_ref is None: - # We expect the original op intent to have been either modified or left untouched, thus must be among - # the returned op intents. If additionally the op intent corresponds to an active flight, we fail a - # different appropriate check. Exception made if skip_if_not_found=True and op intent was deleted: step - # is skipped. - modified_oi_ref = self._find_after_oi(self._orig_oi_ref.id) - if modified_oi_ref is None: - if not skip_if_not_found: - if ( - flight_intent.operational_intent.state - == OperationalIntentState.Activated - ): - with self._scenario.check( - "Operational intent for active flight not deleted", - [self._flight_planner.participant_id], - ) as active_flight_check: - active_flight_check.record_failed( - summary="Operational intent reference for active flight not found in DSS", - severity=Severity.High, - details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", - query_timestamps=[ - self._after_query.request.timestamp - ], - ) - else: - check.record_failed( - summary="Operational intent reference not found in DSS", - severity=Severity.High, - details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by modifying it, but no matching operational intent references were found in the DSS in the area of the flight intent", - query_timestamps=[self._after_query.request.timestamp], - ) - else: - self._scenario.record_note( - self._flight_planner.participant_id, - f"Operational intent reference with ID {self._orig_oi_ref.id} not found in DSS, instructed to skip test step.", - ) - self._scenario.end_test_step() - return None - oi_ref = modified_oi_ref - - else: - # we expect the original op intent to have been replaced with a new one, thus old one must NOT be among the returned op intents - if self._find_after_oi(self._orig_oi_ref.id) is not None: - check.record_failed( - summary="Operational intent reference found duplicated in DSS", - severity=Severity.High, - details=f"USS {self._flight_planner.participant_id} was supposed to have shared with the DSS an updated operational intent by replacing it, but it ended up duplicating the operational intent in the DSS", - query_timestamps=[self._after_query.request.timestamp], - ) - oi_ref = self._new_oi_ref - - goidr_json, oi_full_query = self._dss.get_full_op_intent_without_validation( - oi_ref - ) - self._scenario.record_query(oi_full_query) - with self._scenario.check( - "Operational intent details retrievable", - [self._flight_planner.participant_id], - ) as check: - if oi_full_query.status_code != 200: - check.record_failed( - summary="Operational intent details could not be retrieved from USS", - severity=Severity.High, - details=f"Received status code {oi_full_query.status_code} from {self._flight_planner.participant_id} when querying for details of operational intent {oi_ref.id}", - query_timestamps=[oi_full_query.request.timestamp], - ) - - # if schema validation errors or standard req validation - with self._scenario.check( - "Invalid data in Operational intent details shared by Mock USS for negative test", - [self._flight_planner.participant_id], - ) as check: - - validation_errors = [] - # schema_validation_errors = schema_validation.validate( - # schema_validation.F3548_21.OpenAPIPath, - # schema_validation.F3548_21.GetOperationalIntentDetailsResponse, - # oi_full_query.response.json, - # ) - schema_validation_errors = None - logger.debug(f"Schema validation errors {schema_validation_errors}") - if schema_validation_errors: - details = ( - "The response received from querying operational intent details failed validation against the required OpenAPI schema:\n" - + "\n".join( - f"At {e.json_path} in the response: {e.message}" - for e in schema_validation_errors - ) - ) - validation_errors.append(details) - else: - oi_full = None - try: - goidr = ImplicitDict.parse( - goidr_json, GetOperationalIntentDetailsResponse - ) - oi_full = goidr.operational_intent - - if ( - oi_full.reference.state == OperationalIntentState.Accepted - or oi_full.reference.state == OperationalIntentState.Activated - ) and oi_full.details.get("off_nominal_volumes", None): - details = f"Operational intent {oi_full.reference.id} had {len(oi_full.details.off_nominal_volumes)} off-nominal volumes in wrong state - {oi_full.reference.state}" - validation_errors.append(details) - - def volume_vertices(v4): - if "outline_circle" in v4.volume: - return 1 - if "outline_polygon" in v4.volume: - return len(v4.volume.outline_polygon.vertices) - - all_volumes = oi_full.details.get( - "volumes", [] - ) + oi_full.details.get("off_nominal_volumes", []) - n_vertices = sum(volume_vertices(v) for v in all_volumes) - - if n_vertices > 10000: - details = ( - f"Operational intent {oi_full.reference.id} had too many total vertices - {n_vertices}", - ) - validation_errors.append(details) - except (KeyError, ValueError) as e: - logger.debug( - f"Validation error in GetOperationalIntentDetailsResponse. {e}" - ) - validation_errors.append(e) - - if not validation_errors: - check.record_failed( - summary="This negative test case requires invalid data shared with other USS in Operational intent details ", - severity=Severity.High, - details=f"Data shared by Mock USS with other USSes had no invalid data. This test case required invalid data for testing.", - query_timestamps=[oi_full_query.request.timestamp], - ) - - self._scenario.end_test_step() - return oi_ref diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md index 6581b593da..ce747a72de 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md @@ -1,6 +1,6 @@ # Plan flight Expect Failed test step -This page describes the content of a common test case where a valid user flight intent fails in a flight planner, because of invalid data for a nearby flight shared by another USS. See `plan_flight_intent_expect_failed` in [test_steps.py](invalid_op_test_steps.py). +This page describes the content of a common test case where a valid user flight intent fails in a flight planner, because of invalid data shared for a nearby flight shared by another USS. See `plan_flight_intent_expect_failed` in invalid_op_test_steps.py. ## Plan should fail check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md index e13318769e..8511b562ae 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md @@ -1,6 +1,6 @@ # Validate flight sharing invalid data test step -This step verifies that a created flight is shared properly per ASTM F3548-21 by querying the DSS for flights in the area of the flight intent, and then retrieving the details from the USS if the operational intent reference is found. See `validate_shared_operational_intent` in [test_steps.py](invalid_op_test_steps.py). +This step verifies that a created flight is shared properly per ASTM F3548-21 by querying the DSS for flights in the area of the flight intent, and then retrieving the details from the USS if the operational intent reference is found. See `expect_shared_with_invalid_data` in invalid_op_test_steps.py. ## DSS responses check From 246cd3ff7c7a3715ef9ee065df5fff5cd6b11119 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Tue, 31 Oct 2023 13:29:02 -0700 Subject: [PATCH 10/27] Per PR review, created a new requirement for no notifications --- .../interuss/f3548/notification_requirements.md | 9 +++++++++ .../validate_no_notification_operational_intent.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md diff --git a/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md b/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md new file mode 100644 index 0000000000..d0cff3adda --- /dev/null +++ b/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md @@ -0,0 +1,9 @@ +# ASTM F3548-21 notifications requirement + +While ASTM F3548-21 does not specifically prohibit sending notifications for operational intent changes when those notifications were not specifically prescribed by a notification that an operational intent may be relevant to a subscription (response from the DSS), InterUSS expects that notifications will only be sent at these particular times. +SCD0085 requirement states only a positive requirement (preconditions satisified => notification must be sent). +It doesn't establish a negative requirement (preconditions not satisfied => no notifications may be sent). +As these negative requirements are reasonable and related to SCD0085, interUSS has added it as a requirement for testing. +This can happen in following situations - + - If a USS was unable to write an entity reference to the DSS, it should not erroneously notify that operational intent, to another USS subscribed in the area. + - A USS wrote an entity reference to the DSS, and was not notified of any other USS subscription, so it should not send a notification to any USS. So the USS shall (OnlyPrescribedNotifications) only send operational intent notifications when prescribed by SCD0085. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md index e534f77f39..3cabd872fa 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md @@ -7,4 +7,4 @@ This step verifies when a flight is not created, it is also not notified by chec ## Expect Notification not sent check -**[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** +**[interuss.f3548.notification_requirements](../../../../../requirements/interuss/f3548/notification_requirements.md)** From d322d0ac6294ee310434fb8dedb6572838091965 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Tue, 31 Oct 2023 14:17:43 -0700 Subject: [PATCH 11/27] Adding empty sceanrio and fixing documentation links --- .../f3548/notification_requirements.md | 4 +- .../get_op_data_validation.py | 55 +++++++++++++++++++ .../validate_get_operational_intent.md | 2 +- ...date_no_notification_operational_intent.md | 4 +- ...alidate_notification_operational_intent.md | 2 +- 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py diff --git a/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md b/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md index d0cff3adda..a3535e725e 100644 --- a/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md +++ b/monitoring/uss_qualifier/requirements/interuss/f3548/notification_requirements.md @@ -5,5 +5,5 @@ SCD0085 requirement states only a positive requirement (preconditions satisified It doesn't establish a negative requirement (preconditions not satisfied => no notifications may be sent). As these negative requirements are reasonable and related to SCD0085, interUSS has added it as a requirement for testing. This can happen in following situations - - - If a USS was unable to write an entity reference to the DSS, it should not erroneously notify that operational intent, to another USS subscribed in the area. - - A USS wrote an entity reference to the DSS, and was not notified of any other USS subscription, so it should not send a notification to any USS. So the USS shall (OnlyPrescribedNotifications) only send operational intent notifications when prescribed by SCD0085. +* NoDssEntityNoNotification If a USS was unable to write an entity reference to the DSS, it should not erroneously notify that operational intent, to another USS subscribed in the area. +* NoSubscriptionNoNotification If a USS wrote an entity reference to the DSS, and was notified of no other USS subscription, then it should not send a notification to any USS. So the USS shall (OnlyPrescribedNotifications) only send operational intent notifications when prescribed by SCD0085. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py new file mode 100644 index 0000000000..d660e2fef1 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -0,0 +1,55 @@ +from typing import Optional + +from monitoring.uss_qualifier.resources.astm.f3548.v21 import DSSInstanceResource +from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance +from monitoring.uss_qualifier.resources.flight_planning import ( + FlightIntentsResource, +) +from monitoring.uss_qualifier.resources.flight_planning.flight_intent import ( + FlightIntent, +) +from monitoring.uss_qualifier.resources.flight_planning.flight_planner import ( + FlightPlanner, +) +from monitoring.uss_qualifier.resources.flight_planning.flight_planners import ( + FlightPlannerResource, +) + +from monitoring.uss_qualifier.scenarios.scenario import ( + TestScenario, +) + +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( + MockUSSClient, + MockUSSResource, +) + +class GetOpResponseDataValidationByUSS(TestScenario): + flight_1_planned_time_range_A: FlightIntent + + flight_2_planned_time_range_A: FlightIntent + + tested_uss: FlightPlanner + control_uss: MockUSSClient + dss: DSSInstance + + def __init__( + self, + tested_uss: FlightPlannerResource, + control_uss: MockUSSResource, + dss: DSSInstanceResource, + flight_intents: Optional[FlightIntentsResource] = None, + ): + super().__init__() + self.tested_uss = tested_uss.flight_planner + self.control_uss = control_uss.mock_uss + self.dss = dss.dss + + + + def run(self): + self.begin_test_scenario() + pass + self.end_test_scenario() + + diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md index bd97d824e3..e65527b0b0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_get_operational_intent.md @@ -3,7 +3,7 @@ This step verifies that a USS makes a GET request to get the intent_details of an existing operation when needed as per ASTM F3548-21 by checking the interuss interactions of mock uss ## MockUSS interactions request check -**[interuss.mock_uss.interactions.Interactions](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. ## Expect GET request check **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md index 3cabd872fa..e084bba35d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_no_notification_operational_intent.md @@ -3,8 +3,8 @@ This step verifies when a flight is not created, it is also not notified by checking the interuss interactions of mock_uss instance. ## MockUSS interactions request check -**[interuss.mock_uss.interactions.Interactions](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. ## Expect Notification not sent check -**[interuss.f3548.notification_requirements](../../../../../requirements/interuss/f3548/notification_requirements.md)** +**[interuss.f3548.notification_requirements.NoDssEntityNoNotification](../../../../../requirements/interuss/f3548/notification_requirements.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md index 26ff33ec32..5fa28e5031 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_notification_operational_intent.md @@ -3,7 +3,7 @@ This step verifies that, when creating or modifying an operational intent, a USS sent the required notification for a relevant subscription owned by a mock_uss instance by checking the interactions of that mock_uss instance. ## MockUSS interactions request check -**[interuss.mock_uss.interactions.Interactions](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. ## Expect Notification sent check **[astm.f3548.v21.SCD0085](../../../../../requirements/astm/f3548/v21.md)** From 493247b72a39b07318faf3ba842231f2c852d623 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Tue, 31 Oct 2023 15:18:19 -0700 Subject: [PATCH 12/27] Fix format --- .../utm/data_exchange_validation/get_op_data_validation.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index d660e2fef1..a2293a2d41 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -24,6 +24,7 @@ MockUSSResource, ) + class GetOpResponseDataValidationByUSS(TestScenario): flight_1_planned_time_range_A: FlightIntent @@ -45,11 +46,7 @@ def __init__( self.control_uss = control_uss.mock_uss self.dss = dss.dss - - def run(self): self.begin_test_scenario() pass self.end_test_scenario() - - From 9f46935d7b630826117636f16d4bfee3fbd537bc Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Tue, 31 Oct 2023 23:57:39 -0700 Subject: [PATCH 13/27] Fix the requirement reference in documentation --- .../plan_flight_intent_expect_failed.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md index ce747a72de..e55838a6c6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md @@ -4,20 +4,21 @@ This page describes the content of a common test case where a valid user flight ## Plan should fail check -Flight intent data of a nearby flight shared was invalid, therefore it should have been failed to plan by the USS per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +A USS shouldn't go ahead and plan if it doesn't have accurate information. +As per SCD0035 a USS needs to verify a particular conflict status. +If flight intent data shared for a nearby flight shared is invalid, the USS can't successfully perform that verification. +It couldn't have obtained valid information about the other operational intents it's supposed to verify that conflict status against. +Therefore, the USS should fail an attempt to plan. If the plan succeeds, we know they've violated SCD0035 because they failed to verify that particular conflict status. -## Failure If Planned check +**[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** -Flight intent data of a nearby flight shared was invalid, but the result was planned. It should have been been failed. -If the USS indicates that the injection did not fail, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +## Failure if Planned check -## Failure If Conflict check -Flight intent data of a nearby flight shared was invalid, but the result was conflict with flight. It should have been been failed. +Flight intent data of a nearby flight shared was invalid, but the result was planned. It should have been been failed. If the USS indicates that the injection did not fail, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. +**[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** -## Failure If Rejected check +## Failure if Rejected check Flight intent data of a nearby flight shared was invalid, but the result rejected. It should have been been failed. If the USS indicates that the injection did not fail, this check will fail per From be300fa6a02acd992d22713ccbaab53c10c162db Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 00:03:24 -0700 Subject: [PATCH 14/27] Documentation change per PR --- .../astm/utm/data_exchange_validation/get_op_data_validation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index d3371833f0..56f38bb029 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -1,4 +1,4 @@ -# SCD Data Validation of GET operational intents by USS test scenario +# Data Validation of GET operational intents by USS test scenario ## Description This test checks that the USS being tested validates the operational intents received as response to its GET request from another USS. From 52f130895a78b0f4bafac88be514a0a8bfb45f9f Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 00:13:01 -0700 Subject: [PATCH 15/27] Removed requirement reference from test step, as per PR review --- .../utm/data_exchange_validation/get_op_data_validation.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 56f38bb029..73e6970fe9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -53,7 +53,6 @@ If this notification was pushed, the GET operational intent data validation test ### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 on time range A via the tested USS. It checks if any conflicts with flight 2 which is of equal priority and came first. -per **[astm.f3548.v21.SCD0035](../../../../requirements/astm/f3548/v21.md)**. ### [Validate flight 1 sharing test step](../validate_shared_operational_intent.md) Validate flight 1 is planned. @@ -63,8 +62,7 @@ Tested_uss needs to make GET request for obtaining details of flight 2. In a previous step, we checked there was no notification of flight 2 to tested_uss. ### [Validate flight1 Notification sent to Control_uss test step](test_steps/validate_notification_operational_intent.md) -Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. As per **[astm.f3548.v21.USS0105](../../../../requirements/astm/f3548/v21.md)** and **[astm.f3548.v21.SCD0085](../../../../requirements/astm/f3548/v21.md)** - +Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. ### [Delete tested_uss flight test step](../../../flight_planning/delete_flight_intent.md) Teardown @@ -86,7 +84,6 @@ If this notification was pushed, the GET operational intent data validation test ### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 on time range A via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. -per **[astm.f3548.v21.SCD0035](../../../../requirements/astm/f3548/v21.md)**. ### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) Validate flight 1 is not shared. From 42f337ad58513c399faf80c4e8601a242da56a87 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 00:18:02 -0700 Subject: [PATCH 16/27] Added more description to test steps --- .../data_exchange_validation/get_op_data_validation.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 73e6970fe9..e62db97df4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -63,6 +63,7 @@ In a previous step, we checked there was no notification of flight 2 to tested_u ### [Validate flight1 Notification sent to Control_uss test step](test_steps/validate_notification_operational_intent.md) Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. + ### [Delete tested_uss flight test step](../../../flight_planning/delete_flight_intent.md) Teardown @@ -79,23 +80,24 @@ Validate that flight 2 is shared with invalid data as a modified behavior is inj ### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) There should be no subscription by tested_uss to trigger notification of flight 2. This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. -If this notification was pushed, the GET operational intent data validation test cannot be done. +If this notification was pushed, the GET operational intent data validation test cannot be carried out. ### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 on time range A via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. ### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) -Validate flight 1 is not shared. +Validate flight 1 is not shared with DSS, as plan failed. ### [Validate flight 2 GET interaction test step](test_steps/validate_get_operational_intent.md) Tested_uss needs to make GET request for obtaining details of flight 2 from control_uss. In a previous step, we checked there was no notification of flight 2 to tested_uss. +Hence USS will have to obtain details using GET request. ### [Validate flight 1 Notification not sent to Control_uss test step](test_steps/validate_no_notification_operational_intent.md) ### [Delete Control_uss flight test step](../../../flight_planning/delete_flight_intent.md) - +Teardown ## Cleanup ### Successful flight deletion check From bfda3e3b806413a44035f4547e80e3d72e54d850 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 00:37:11 -0700 Subject: [PATCH 17/27] Fix per PR review --- .../get_op_data_validation.md | 7 ++++--- .../test_steps/plan_flight_intent_expect_failed.md | 11 ----------- ...erational_intent_but_with_invalid_interuss_data.md | 5 ++++- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index e62db97df4..4f3cec80e4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -71,10 +71,11 @@ Teardown Teardown ## Flight planning prevented due to invalid data sharing test case -### [Control_uss plans flight 2, sharing invalid interuss data test step](../../../flight_planning/plan_flight_intent.md) -Flight 2 on time range A should be successfully planned by the control USS. +### [Control_uss plans flight 2, sharing invalid operational intent data test step](../../../flight_planning/plan_flight_intent.md) +Flight 2 on time range A should be successfully planned by the control_uss. +The control_uss, which is mock_uss is instructed to share invalid data with other USS, for negative test. -### [Validate flight 2 shared intent with invalid interuss data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) +### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. ### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md index e55838a6c6..34e572c527 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/plan_flight_intent_expect_failed.md @@ -12,14 +12,3 @@ Therefore, the USS should fail an attempt to plan. If the plan succeeds, we know **[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** -## Failure if Planned check - -Flight intent data of a nearby flight shared was invalid, but the result was planned. It should have been been failed. -If the USS indicates that the injection did not fail, this check will fail per -**[astm.f3548.v21.SCD0035](../../../../../requirements/astm/f3548/v21.md)** - -## Failure if Rejected check - -Flight intent data of a nearby flight shared was invalid, but the result rejected. It should have been been failed. -If the USS indicates that the injection did not fail, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../../requirements/interuss/automated_testing/flight_planning.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md index 8511b562ae..38473df15c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md @@ -16,5 +16,8 @@ If the operational intent details for the flight cannot be retrieved from the US ## Invalid data in Operational intent details shared by Mock USS for negative test check -Mock USS shares operational intent details response for the negative test case as per [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120) but the data is invalid, this check fill fail per **[astm.f3548.v21.USS0105](../../../../../requirements/astm/f3548/v21.md)**. +Mock USS shares operational intent details response for the negative test case as per [the GetOperationalIntentDetailsResponse schema of the OpenAPI specification](https://github.com/astm-utm/Protocol/blob/v1.0.0/utm.yaml#L1120). +If the operational intent details from mock_uss is valid, this check will fail. It would mean mock_uss is not behaving correctly. +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + From 43c2b3d111cc3e7209b3d28ed61aa2000f8263dc Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 13:03:40 -0700 Subject: [PATCH 18/27] Removed time_range suffix for flight intents, as per PR comment --- .../utm/data_exchange_validation/get_op_data_validation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 4f3cec80e4..56c4326b4f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -13,8 +13,8 @@ Notably the following requirements: FlightIntentsResource provides the two V-shaped flight intents. The convex hulls of the 2D footprints of the two flights intersect, but the polygons do not intersect. There is an overlap in time and altitude of the two flights. -- flight_1_planned_time_range_A -- flight_2_planned_time_range_A +- flight_1 +- flight_2 ### control_uss MockUSSResource that will be used for planning flights, controlling data shared for validation testing, and gathering interuss interactions from mock_uss. From 4a3df76ab472daacf6cfb0916f42f6b5c2eb5c78 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 13:07:38 -0700 Subject: [PATCH 19/27] Remover time_range_A suffix from flight intent --- .../usa/kentland/flight_intents/non_conflicting.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml index af7d58d10e..c492411f23 100644 --- a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml +++ b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml @@ -1,5 +1,5 @@ intents: - flight_1_planned_time_range_A: + flight_1: full: basic_information: usage_state: Planned @@ -49,7 +49,7 @@ intents: uas_type_certificate: '' - flight_2_planned_time_range_A: + flight_2: full: basic_information: usage_state: Planned From ae5427901a6dabe512456254b3644a66fd261d7d Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 15:24:57 -0700 Subject: [PATCH 20/27] Revert "Remover time_range_A suffix from flight intent" This reverts commit 4a3df76ab472daacf6cfb0916f42f6b5c2eb5c78. --- .../usa/kentland/flight_intents/non_conflicting.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml index c492411f23..af7d58d10e 100644 --- a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml +++ b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml @@ -1,5 +1,5 @@ intents: - flight_1: + flight_1_planned_time_range_A: full: basic_information: usage_state: Planned @@ -49,7 +49,7 @@ intents: uas_type_certificate: '' - flight_2: + flight_2_planned_time_range_A: full: basic_information: usage_state: Planned From 9dc09125998f2679adc8ad2614fcb836b7ad8f09 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Wed, 1 Nov 2023 15:27:34 -0700 Subject: [PATCH 21/27] Removed time_range_A suffix from flight intents --- .../usa/kentland/flight_intents/non_conflicting.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml index af7d58d10e..c492411f23 100644 --- a/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml +++ b/monitoring/uss_qualifier/test_data/usa/kentland/flight_intents/non_conflicting.yaml @@ -1,5 +1,5 @@ intents: - flight_1_planned_time_range_A: + flight_1: full: basic_information: usage_state: Planned @@ -49,7 +49,7 @@ intents: uas_type_certificate: '' - flight_2_planned_time_range_A: + flight_2: full: basic_information: usage_state: Planned From e58a0fe1a5c1bda0176a44f81df54da706f0f27d Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 2 Nov 2023 09:33:58 -0700 Subject: [PATCH 22/27] Removing unreferenced file and description per PR review --- .../interuss/mock_uss/interactions.md | 9 --------- .../get_op_data_validation.md | 16 ++++++++-------- 2 files changed, 8 insertions(+), 17 deletions(-) delete mode 100644 monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md diff --git a/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md b/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md deleted file mode 100644 index f46ca6351c..0000000000 --- a/monitoring/uss_qualifier/requirements/interuss/mock_uss/interactions.md +++ /dev/null @@ -1,9 +0,0 @@ -InterUss mock_uss interactions requirements - -## Overview -Mock_uss records interactions on its uss/v1/operational_intents endpoints. It provides an endpoint for uss_qualifier to get these interactions for testing various scenarios. - -## Requirements - -### Interactions -GET /mock_uss/interuss_logging/logs returns a list of interactions that took place between the time specified by query parameter from_time and the current time. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 56c4326b4f..08162d5162 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -40,18 +40,18 @@ Both USSs are requested to remove all flights from the area under test. ## Successfully plan flight near an existing flight test case ### [Control_uss plans flight 2 test step](../../../flight_planning/plan_flight_intent.md) -Flight 2 on time range A should be successfully planned by the control USS. +Flight 2 should be successfully planned by the control USS. ### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) Validate that flight 2 is planned -### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) +### [Validate no notification pushed for flight 2](test_steps/validate_no_notification_operational_intent.md) There should be no subscription by tested_uss to trigger notification of flight 2. This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. If this notification was pushed, the GET operational intent data validation test cannot be done. ### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) -The test driver attempts to plan flight 1 on time range A via the tested USS. It checks if any conflicts with flight 2 +The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 which is of equal priority and came first. ### [Validate flight 1 sharing test step](../validate_shared_operational_intent.md) @@ -59,7 +59,7 @@ Validate flight 1 is planned. ### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) Tested_uss needs to make GET request for obtaining details of flight 2. -In a previous step, we checked there was no notification of flight 2 to tested_uss. +In a previous step(Validate no notification pushed for flight 2), we checked there was no notification of flight 2 to tested_uss. ### [Validate flight1 Notification sent to Control_uss test step](test_steps/validate_notification_operational_intent.md) Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. @@ -72,19 +72,19 @@ Teardown ## Flight planning prevented due to invalid data sharing test case ### [Control_uss plans flight 2, sharing invalid operational intent data test step](../../../flight_planning/plan_flight_intent.md) -Flight 2 on time range A should be successfully planned by the control_uss. +Flight 2 should be successfully planned by the control_uss. The control_uss, which is mock_uss is instructed to share invalid data with other USS, for negative test. ### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. -### [Validate no notification pushed for flight 2](../validate_not_shared_operational_intent.md) +### [Validate no notification pushed for flight 2](test_steps/validate_no_notification_operational_intent.md) There should be no subscription by tested_uss to trigger notification of flight 2. This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. If this notification was pushed, the GET operational intent data validation test cannot be carried out. ### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) -The test driver attempts to plan the flight 1 on time range A via the tested_uss. It checks if any conflicts with flight 2 +The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. ### [Validate flight 1 not shared by tested_uss test step](../validate_not_shared_operational_intent.md) @@ -92,7 +92,7 @@ Validate flight 1 is not shared with DSS, as plan failed. ### [Validate flight 2 GET interaction test step](test_steps/validate_get_operational_intent.md) Tested_uss needs to make GET request for obtaining details of flight 2 from control_uss. -In a previous step, we checked there was no notification of flight 2 to tested_uss. +In a previous step(Validate no notification pushed for flight 2), we checked there was no notification of flight 2 to tested_uss. Hence USS will have to obtain details using GET request. ### [Validate flight 1 Notification not sent to Control_uss test step](test_steps/validate_no_notification_operational_intent.md) From 2b68ef5a9d1ce7cf523c597f3ea99f92257538b7 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 2 Nov 2023 13:07:45 -0700 Subject: [PATCH 23/27] fixing variable name per PR reveiw --- .../utm/data_exchange_validation/get_op_data_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index a2293a2d41..063b1677d1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -26,9 +26,9 @@ class GetOpResponseDataValidationByUSS(TestScenario): - flight_1_planned_time_range_A: FlightIntent + flight_1: FlightIntent - flight_2_planned_time_range_A: FlightIntent + flight_2: FlightIntent tested_uss: FlightPlanner control_uss: MockUSSClient From dbcb318f6f46e7274ab907ef5525c37381b4f266 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 2 Nov 2023 14:28:56 -0700 Subject: [PATCH 24/27] Removed link for no notification pushed --- .../get_op_data_validation.md | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 08162d5162..d5b916605c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -46,10 +46,19 @@ Flight 2 should be successfully planned by the control USS. Validate that flight 2 is planned ### [Validate no notification pushed for flight 2](test_steps/validate_no_notification_operational_intent.md) -There should be no subscription by tested_uss to trigger notification of flight 2. -This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. +Check there is no subscription by tested_uss to trigger notification of flight 2. +If no notification is pushed by control_uss to tested_uss, we know tested_uss has no subscription. +This will ensure that while planning a nearby flight, tested_uss will need to make a GET request to control_uss for flight 2 details. If this notification was pushed, the GET operational intent data validation test cannot be done. +#### MockUSS interactions request check +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + +#### Expect Notification not sent check +As there is no subscription expected, no notification of operational_intent should be sent. +**[interuss.f3548.notification_requirements.NoSubscriptionNoNotification](../../../../requirements/interuss/f3548/notification_requirements.md)** + + ### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 which is of equal priority and came first. @@ -79,9 +88,18 @@ The control_uss, which is mock_uss is instructed to share invalid data with othe Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. ### [Validate no notification pushed for flight 2](test_steps/validate_no_notification_operational_intent.md) -There should be no subscription by tested_uss to trigger notification of flight 2. -This will ensure that while planning a nearby flight tested_uss will need to make a GET request to control_uss for flight 2 details. -If this notification was pushed, the GET operational intent data validation test cannot be carried out. +Check there is no subscription by tested_uss to trigger notification of flight 2. +If no notification is pushed by control_uss to tested_uss, we know tested_uss has no subscription. +This will ensure that while planning a nearby flight, tested_uss will need to make a GET request to control_uss for flight 2 details. +If this notification was pushed, the GET operational intent data validation test cannot be done. + +#### MockUSS interactions request check +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. + +#### Expect Notification not sent check +As there is no subscription expected, no notification of operational_intent should be sent. +**[interuss.f3548.notification_requirements.NoSubscriptionNoNotification](../../../../requirements/interuss/f3548/notification_requirements.md)** + ### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 From 4ae2814bdd04e9834bb849de795efa43f0e5aa0e Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 2 Nov 2023 15:08:04 -0700 Subject: [PATCH 25/27] Fixed removing link --- .../utm/data_exchange_validation/get_op_data_validation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index d5b916605c..67cb7895fc 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -45,7 +45,7 @@ Flight 2 should be successfully planned by the control USS. ### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) Validate that flight 2 is planned -### [Validate no notification pushed for flight 2](test_steps/validate_no_notification_operational_intent.md) +### Validate no notification pushed for flight 2 test step Check there is no subscription by tested_uss to trigger notification of flight 2. If no notification is pushed by control_uss to tested_uss, we know tested_uss has no subscription. This will ensure that while planning a nearby flight, tested_uss will need to make a GET request to control_uss for flight 2 details. @@ -87,7 +87,7 @@ The control_uss, which is mock_uss is instructed to share invalid data with othe ### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. -### [Validate no notification pushed for flight 2](test_steps/validate_no_notification_operational_intent.md) +### Validate no notification pushed for flight 2 test step Check there is no subscription by tested_uss to trigger notification of flight 2. If no notification is pushed by control_uss to tested_uss, we know tested_uss has no subscription. This will ensure that while planning a nearby flight, tested_uss will need to make a GET request to control_uss for flight 2 details. From 5dc237586439275023564e08d96ecebbd760ce72 Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 2 Nov 2023 16:01:15 -0700 Subject: [PATCH 26/27] Fix link --- .../utm/data_exchange_validation/get_op_data_validation.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 67cb7895fc..cb13e56d4f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -52,13 +52,12 @@ This will ensure that while planning a nearby flight, tested_uss will need to ma If this notification was pushed, the GET operational intent data validation test cannot be done. #### MockUSS interactions request check -**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. #### Expect Notification not sent check As there is no subscription expected, no notification of operational_intent should be sent. **[interuss.f3548.notification_requirements.NoSubscriptionNoNotification](../../../../requirements/interuss/f3548/notification_requirements.md)** - ### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 which is of equal priority and came first. @@ -94,13 +93,12 @@ This will ensure that while planning a nearby flight, tested_uss will need to ma If this notification was pushed, the GET operational intent data validation test cannot be done. #### MockUSS interactions request check -**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../../requirements/interuss/mock_uss/hosted_instance.md)**. +**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. #### Expect Notification not sent check As there is no subscription expected, no notification of operational_intent should be sent. **[interuss.f3548.notification_requirements.NoSubscriptionNoNotification](../../../../requirements/interuss/f3548/notification_requirements.md)** - ### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 which is of equal priority and came first. From 23141b2bf6fe57191301111edb8fc2906a1546cc Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Fri, 3 Nov 2023 00:10:18 -0700 Subject: [PATCH 27/27] Removing check from a precondition step as per PR review --- .../get_op_data_validation.md | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index cb13e56d4f..642b1fd1bb 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -39,24 +39,19 @@ Both USSs are requested to remove all flights from the area under test. **[interuss.automated_testing.flight_planning.ClearArea](../../../../requirements/interuss/automated_testing/flight_planning.md)** ## Successfully plan flight near an existing flight test case + ### [Control_uss plans flight 2 test step](../../../flight_planning/plan_flight_intent.md) Flight 2 should be successfully planned by the control USS. ### [Validate flight 2 sharing test step](../validate_shared_operational_intent.md) Validate that flight 2 is planned -### Validate no notification pushed for flight 2 test step -Check there is no subscription by tested_uss to trigger notification of flight 2. -If no notification is pushed by control_uss to tested_uss, we know tested_uss has no subscription. -This will ensure that while planning a nearby flight, tested_uss will need to make a GET request to control_uss for flight 2 details. -If this notification was pushed, the GET operational intent data validation test cannot be done. - -#### MockUSS interactions request check -**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. - -#### Expect Notification not sent check -As there is no subscription expected, no notification of operational_intent should be sent. -**[interuss.f3548.notification_requirements.NoSubscriptionNoNotification](../../../../requirements/interuss/f3548/notification_requirements.md)** +### Precondition - check tested_uss has no subscription in flight 2 area test step +In order to run this test scenario, we need tested_uss to trigger GET operational intent request to mock_uss. +So we need to make sure that there is no subscription of tested_uss, that would trigger notification of flight 2 to tested_uss. +No notification pushed by control_uss to tested_uss, will ensure that tested_uss will make a GET request to control_uss for flight 2 details, +while planning a nearby flight. +If a notification is sent to tested_uss, the precondition for running this scenario will not be satisfied. ### [Tested_uss plans flight 1 test step](../../../flight_planning/plan_flight_intent.md) The test driver attempts to plan flight 1 via the tested USS. It checks if any conflicts with flight 2 @@ -66,8 +61,9 @@ which is of equal priority and came first. Validate flight 1 is planned. ### [Validate flight2 GET interaction test step](test_steps/validate_get_operational_intent.md) -Tested_uss needs to make GET request for obtaining details of flight 2. -In a previous step(Validate no notification pushed for flight 2), we checked there was no notification of flight 2 to tested_uss. +Validate that tested_uss makes a GET request for obtaining details of flight 2 from control_uss. +In a previous step (Precondition - check tested_uss has no subscription in flight 2 area), we ensured that no notification of flight 2 was sent to tested_uss. +Hence, tested_uss will need to make a GET request to obtain flight 2 details. ### [Validate flight1 Notification sent to Control_uss test step](test_steps/validate_notification_operational_intent.md) Tested_uss notifies flight 1 to Control_uss, due to its subscription through flight 2. @@ -86,18 +82,12 @@ The control_uss, which is mock_uss is instructed to share invalid data with othe ### [Validate flight 2 shared operational intent with invalid data test step](test_steps/validate_sharing_operational_intent_but_with_invalid_interuss_data.md) Validate that flight 2 is shared with invalid data as a modified behavior is injected by uss_qualifier for a negative test. -### Validate no notification pushed for flight 2 test step -Check there is no subscription by tested_uss to trigger notification of flight 2. -If no notification is pushed by control_uss to tested_uss, we know tested_uss has no subscription. -This will ensure that while planning a nearby flight, tested_uss will need to make a GET request to control_uss for flight 2 details. -If this notification was pushed, the GET operational intent data validation test cannot be done. - -#### MockUSS interactions request check -**[interuss.mock_uss.hosted_instance.ExposeInterface](../../../../requirements/interuss/mock_uss/hosted_instance.md)**. - -#### Expect Notification not sent check -As there is no subscription expected, no notification of operational_intent should be sent. -**[interuss.f3548.notification_requirements.NoSubscriptionNoNotification](../../../../requirements/interuss/f3548/notification_requirements.md)** +### Precondition - check tested_uss has no subscription in flight 2 area test step +In order to run this test scenario, we need tested_uss to trigger GET operational intent request to mock_uss. +So we need to make sure that there is no subscription of tested_uss, that would trigger notification of flight 2 to tested_uss. +No notification pushed by control_uss to tested_uss, will ensure that tested_uss will make a GET request to control_uss for flight 2 details, +while planning a nearby flight. +If a notification is sent to tested_uss, the precondition for running this scenario will not be satisfied. ### [Test_uss attempts to plan flight 1, expect failure test step](test_steps/plan_flight_intent_expect_failed.md) The test driver attempts to plan the flight 1 via the tested_uss. It checks if any conflicts with flight 2 @@ -107,9 +97,9 @@ which is of equal priority and came first. Validate flight 1 is not shared with DSS, as plan failed. ### [Validate flight 2 GET interaction test step](test_steps/validate_get_operational_intent.md) -Tested_uss needs to make GET request for obtaining details of flight 2 from control_uss. -In a previous step(Validate no notification pushed for flight 2), we checked there was no notification of flight 2 to tested_uss. -Hence USS will have to obtain details using GET request. +Validate that tested_uss makes a GET request for obtaining details of flight 2 from control_uss. +In a previous step (Precondition - check tested_uss has no subscription in flight 2 area), we ensured that no notification of flight 2 was sent to tested_uss. +Hence, tested_uss will need to make a GET request to obtain flight 2 details. ### [Validate flight 1 Notification not sent to Control_uss test step](test_steps/validate_no_notification_operational_intent.md)