From ea07fd956d044c838cc072b6a77cd10636a6a63c Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Tue, 31 Oct 2023 09:46:27 -0700 Subject: [PATCH] [uss_qualifier] Call flight planner from general flight authorization scenario (#297) * Call flight planner from general flight authorization scenario * Fix import location --- github_pages/static/index.md | 5 + .../clients/flight_planning/client_v1.py | 39 +++- monitoring/monitorlib/fetch/__init__.py | 14 ++ .../dev/general_flight_auth.yaml | 21 ++ .../configurations/dev/library/resources.yaml | 158 ++++++++------ monitoring/uss_qualifier/reports/report.py | 6 +- .../automated_testing/flight_planning.md | 13 +- .../flight_planning/flight_planner.py | 34 +-- .../flight_planning/flight_planners.py | 5 + .../flight_authorization/definitions.py | 18 +- .../general_flight_authorization.md | 12 ++ .../general_flight_authorization.py | 200 +++++++++++++++--- .../ASTMF354821OpIntentInformation.json | 18 ++ .../flight_info/FlightAuthorisationData.json | 98 +++++++++ .../flight_info/RPAS26FlightDetails.json | 102 +++++++++ .../BasicFlightPlanInformationTemplate.json | 41 ++++ .../FlightInfoTemplate.json | 55 +++++ .../monitoring/monitorlib/fetch/Query.json | 6 +- .../definitions/FlightCheck.json | 20 +- 19 files changed, 730 insertions(+), 135 deletions(-) create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json create mode 100644 schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json diff --git a/github_pages/static/index.md b/github_pages/static/index.md index 02cd5b3ce0..9b9bf2290b 100644 --- a/github_pages/static/index.md +++ b/github_pages/static/index.md @@ -33,3 +33,8 @@ These reports were generated during continuous integration for the most recent P * [Sequence view](./artifacts/uss_qualifier/reports/dss_probing/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/dss_probing/requirements) + +### [General flight authorization configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml) + +* [Sequence view](./artifacts/uss_qualifier/reports/general_flight_auth/sequence) +* [Tested requirements](./artifacts/uss_qualifier/reports/general_flight_auth/requirements) diff --git a/monitoring/monitorlib/clients/flight_planning/client_v1.py b/monitoring/monitorlib/clients/flight_planning/client_v1.py index 8fa3cf32bb..ce91c33182 100644 --- a/monitoring/monitorlib/clients/flight_planning/client_v1.py +++ b/monitoring/monitorlib/clients/flight_planning/client_v1.py @@ -19,9 +19,10 @@ from monitoring.monitorlib.clients.flight_planning.planning import ( PlanningActivityResponse, ) -from monitoring.monitorlib.fetch import query_and_describe +from monitoring.monitorlib.fetch import query_and_describe, QueryType from monitoring.monitorlib.geotemporal import Volume4D from monitoring.monitorlib.infrastructure import UTMClientSession +from monitoring.uss_qualifier.configurations.configuration import ParticipantID from uas_standards.interuss.automated_testing.flight_planning.v1 import api from uas_standards.interuss.automated_testing.flight_planning.v1.constants import Scope @@ -29,9 +30,11 @@ class V1FlightPlannerClient(FlightPlannerClient): _session: UTMClientSession + _participant_id: ParticipantID - def __init__(self, session: UTMClientSession): + def __init__(self, session: UTMClientSession, participant_id: ParticipantID): self._session = session + self._participant_id = participant_id def _inject( self, @@ -53,7 +56,13 @@ def _inject( op = api.OPERATIONS[api.OperationID.UpsertFlightPlan] url = op.path.format(flight_plan_id=flight_plan_id) query = query_and_describe( - self._session, op.verb, url, json=req, scope=Scope.Plan + self._session, + op.verb, + url, + json=req, + scope=Scope.Plan, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1UpsertFlightPlan, ) if query.status_code != 200 and query.status_code != 201: raise PlanningActivityError( @@ -108,7 +117,14 @@ def try_end_flight( ) op = api.OPERATIONS[api.OperationID.DeleteFlightPlan] url = op.path.format(flight_plan_id=flight_id) - query = query_and_describe(self._session, op.verb, url, scope=Scope.Plan) + query = query_and_describe( + self._session, + op.verb, + url, + scope=Scope.Plan, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1DeleteFlightPlan, + ) if query.status_code != 200: raise PlanningActivityError( f"Attempt to delete flight plan returned status {query.status_code} rather than 200 as expected", @@ -134,7 +150,12 @@ def try_end_flight( def report_readiness(self) -> TestPreparationActivityResponse: op = api.OPERATIONS[api.OperationID.GetStatus] query = query_and_describe( - self._session, op.verb, op.path, scope=Scope.DirectAutomatedTest + self._session, + op.verb, + op.path, + scope=Scope.DirectAutomatedTest, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1GetStatus, ) if query.status_code != 200: raise PlanningActivityError( @@ -166,7 +187,13 @@ def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: op = api.OPERATIONS[api.OperationID.ClearArea] query = query_and_describe( - self._session, op.verb, op.path, json=req, scope=Scope.DirectAutomatedTest + self._session, + op.verb, + op.path, + json=req, + scope=Scope.DirectAutomatedTest, + participant_id=self._participant_id, + query_type=QueryType.InterUSSFlightPlanningV1ClearArea, ) if query.status_code != 200: raise PlanningActivityError( diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 1661f0eb33..d69a3eb321 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -221,6 +221,20 @@ class QueryType(str, Enum): # InterUSS automated testing versioning interface InterUSSVersioningGetVersion = "interuss.automated_testing.versioning.GetVersion" + # InterUSS automated testing flight_planning interface + InterUSSFlightPlanningV1GetStatus = ( + "interuss.automated_testing.flight_planning.v1.GetStatus" + ) + InterUSSFlightPlanningV1ClearArea = ( + "interuss.automated_testing.flight_planning.v1.ClearArea" + ) + InterUSSFlightPlanningV1UpsertFlightPlan = ( + "interuss.automated_testing.flight_planning.v1.UpsertFlightPlan" + ) + InterUSSFlightPlanningV1DeleteFlightPlan = ( + "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" + ) + @staticmethod def flight_details(rid_version: RIDVersion): if rid_version == RIDVersion.f3411_19: diff --git a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml index eda18dbd56..e1d39e5280 100644 --- a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml +++ b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml @@ -4,13 +4,34 @@ v1: resources: resource_declarations: example_flight_check_table: {$ref: 'library/resources.yaml#/example_flight_check_table'} + + utm_auth: {$ref: 'library/environment.yaml#/utm_auth'} + uss1_flight_planner: {$ref: 'library/environment.yaml#/uss1_flight_planner'} + non_baseline_inputs: + - v1.test_run.resources.resource_declarations.utm_auth + - v1.test_run.resources.resource_declarations.uss1_flight_planner action: test_scenario: scenario_type: scenarios.interuss.flight_authorization.GeneralFlightAuthorization resources: table: example_flight_check_table + planner: uss1_flight_planner artifacts: output_path: output/general_flight_auth raw_report: {} + sequence_view: {} + tested_requirements: + - report_name: requirements + requirement_collections: + example: + requirement_collections: + - requirements: + - REQ_001 + - REQ_002 + - REQ_003 + - REQ_004 + - REQ_007 + participant_requirements: + uss1: example validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index b3667c5a28..0fee0615c5 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -151,73 +151,113 @@ example_flight_check_table: - REQ_002 - REQ_007 description: The first test step defined by the test designer - additional_information: - new_jurisdiction_x: - operation_rule_set: Rules1 - volumes: - - outline_circle: - center: - lng: 7.4774 - lat: 46.9749 - radius: - value: 100 - units: M - altitude_lower: - value: 0 - units: M - reference: SFC - altitude_upper: - value: 100 - units: M - reference: SFC - start_time: - start_of_test: { } - use_timezone: Europe/Berlin - end_time: - offset_from: - starting_from: - next_day: - time_zone: Europe/Zurich - starting_from: - start_of_test: { } - days_of_the_week: [ "Tu", "Th" ] - offset: 12h acceptance_expectation: MustBeAccepted + flight_info: + basic_information: + usage_state: Planned + uas_state: Nominal + area: + - outline_circle: + center: + lng: 7.4774 + lat: 46.9749 + radius: + value: 100 + units: M + altitude_lower: + value: 550 + units: M + # TODO: Change to SFC once mock_uss can process that datum + reference: W84 + altitude_upper: + value: 650 + units: M + # TODO: Change to SFC once mock_uss can process that datum + reference: W84 + start_time: + start_of_test: { } + use_timezone: Europe/Berlin + end_time: + offset_from: + starting_from: + next_day: + time_zone: Europe/Zurich + starting_from: + start_of_test: { } + days_of_the_week: [ "Tu", "Th" ] + offset: 12h + additional_information: + new_jurisdiction_x: + operation_rule_set: Rules1 + # TODO: Remove once mock_uss is fixed to not require U-space flight auth + uspace_flight_authorisation: + uas_serial_number: 1AF49UL5CC5J6K + operation_category: Open + operation_mode: Vlos + uas_class: C0 + identification_technologies: + - ASTMNetRID + connectivity_methods: + - cellular + endurance_minutes: 30 + emergency_procedure_url: https://example.interussplatform.org/emergency + operator_id: CHEo5kut30e0mt01-qwe + uas_id: '' + uas_type_certificate: '' - flight_check_id: TEST_002 requirement_ids: - REQ_001 - REQ_003 - REQ_004 description: The second test step defined by the test designer - additional_information: - new_jurisdiction_x: - operation_rule_set: Rules1 - volumes: - - outline_circle: - center: - lng: 7.4774 - lat: 46.9749 - radius: - value: 100 - units: M - altitude_lower: - value: 50 - units: M - reference: SFC - altitude_upper: - value: 5000 - units: FT - reference: W84 - start_time: - next_day: - time_zone: +02:00 - starting_from: - offset_from: + acceptance_expectation: MustBeAccepted + flight_info: + basic_information: + usage_state: Planned + uas_state: Nominal + area: + - outline_circle: + center: + lng: 7.4774 + lat: 46.9749 + radius: + value: 100 + units: M + altitude_lower: + value: 1424 + units: M + reference: W84 + altitude_upper: + value: 5000 + units: FT + reference: W84 + start_time: + next_day: + time_zone: +02:00 starting_from: - start_of_test: { } - offset: 12h - duration: 5m - conditions_expectation: MustBePresent + offset_from: + starting_from: + start_of_test: { } + offset: 12h + duration: 5m + additional_information: + new_jurisdiction_x: + operation_rule_set: Rules1 + # TODO: Remove once mock_uss is fixed to not require U-space flight auth + uspace_flight_authorisation: + uas_serial_number: 1AF49UL5CC5J6K + operation_category: Open + operation_mode: Vlos + uas_class: C0 + identification_technologies: + - ASTMNetRID + connectivity_methods: + - cellular + endurance_minutes: 30 + emergency_procedure_url: https://example.interussplatform.org/emergency + operator_id: CHEo5kut30e0mt01-qwe + uas_id: '' + uas_type_certificate: '' # ===== Geospatial feature comprehension ===== diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index fa9a110d98..91775d7213 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -457,7 +457,7 @@ def query_passed_checks( def query_failed_checks( self, participant_id: Optional[str] = None - ) -> Iterator[Tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[Tuple[JSONPathExpression, FailedCheck]]: test_suite, test_scenario, action_generator = self.get_applicable_report() if test_suite: report = self.test_suite @@ -486,7 +486,9 @@ def start_time(self) -> Optional[StringBasedDateTime]: @property def end_time(self) -> Optional[StringBasedDateTime]: - return self._conditional(lambda report: report.end_time) + return self._conditional( + lambda report: report.end_time if "end_time" in report else None + ) class AllConditionsEvaluationReport(ImplicitDict): diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md index 63c7169b31..b401fdad94 100644 --- a/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/flight_planning.md @@ -2,19 +2,28 @@ ## Overview -TODO: Link to API YAML and provide overview +When a USS implements the [InterUSS flight_planning automated testing API](https://github.com/interuss/automated_testing_interfaces/tree/main/flight_planning) (or [legacy scd automated testing API](https://github.com/interuss/automated_testing_interfaces/tree/main/scd)), they are expected to respond to requests to that API as defined in the API. Specific requirements are below. ## Requirements -TODO: Describe requirements +### ImplementAPI + +A USS must implement the endpoints defined in the API, accept requests in the data format prescribed in the API, and respond in the data format prescribed in the API. If there is a problem using the API such as a connection error, invalid response code, or invalid data, the USS will have failed to meet this requirement. ### ClearArea +In order to conduct automated tests effectively, the USS must remove all of their existing flights from a particular area when instructed by the test director. This is not an action performed on behalf of an emulated user, but rather an action performed in any way appropriate to support automated testing -- therefore, fulfilling this request may cause actions on the implementing USS's system that no normal user would be able to perform. + ### ExpectedBehavior +When the test director (client of the flight planning API; usually uss_qualifier) requests that a flight planning activity be performed, the API implementer must act as if this request is coming from a normal user attempting to use the USS's system normally. The USS must fulfill this request as it would for a normal user, and these actions are generally expected to succeed (allowing the user to fly) when a UTM rule does not prohibit them. + ### FlightCoveredByOperationalIntent + For InterUSS to effectively test the requirements of ASTM F3548-21, a USS under test must act as if there is a regulatory requirement requiring all flights it manages to provide operational intents according to ASTM F3548-21 at all times for all flights it manages. ### DeleteFlightSuccess + +In order to conduct automated tests effectively, the USS must remove a particular flight when instructed by the test director. This is not an action performed on behalf of an emulated user, but rather an action performed in any way appropriate to support automated testing -- therefore, fulfilling this request may cause actions on the implementing USS's system that no normal user would be able to perform. diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py index 70d1aa2cb7..3ca0a44645 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py @@ -3,7 +3,10 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import infrastructure, fetch -from monitoring.monitorlib.clients.flight_planning.client import PlanningActivityError +from monitoring.monitorlib.clients.flight_planning.client import ( + PlanningActivityError, + FlightPlannerClient, +) from monitoring.monitorlib.clients.flight_planning.client_scd import ( SCDFlightPlannerClient, ) @@ -32,7 +35,6 @@ DeleteFlightResponse, InjectFlightRequest, ClearAreaResponse, - ClearAreaRequest, OperationalIntentState, ClearAreaOutcome, ) @@ -70,6 +72,23 @@ def __init__(self, *args, **kwargs): except ValueError: raise ValueError("FlightPlannerConfiguration.v1_base_url must be a URL") + def to_client( + self, auth_adapter: infrastructure.AuthAdapter + ) -> FlightPlannerClient: + if "scd_injection_base_url" in self and self.scd_injection_base_url: + session = infrastructure.UTMClientSession( + self.scd_injection_base_url, auth_adapter, self.timeout_seconds + ) + return SCDFlightPlannerClient(session) + elif "v1_base_url" in self and self.v1_base_url: + session = infrastructure.UTMClientSession( + self.v1_base_url, auth_adapter, self.timeout_seconds + ) + return V1FlightPlannerClient(session, self.participant_id) + raise ValueError( + "Could not construct FlightPlannerClient from provided configuration" + ) + class FlightPlanner: """Manages the state and the interactions with flight planner USS. @@ -82,16 +101,7 @@ def __init__( auth_adapter: infrastructure.AuthAdapter, ): self.config = config - if "scd_injection_base_url" in config and config.scd_injection_base_url: - session = infrastructure.UTMClientSession( - self.config.scd_injection_base_url, auth_adapter, config.timeout_seconds - ) - self.client = SCDFlightPlannerClient(session) - elif "v1_base_url" in config and config.v1_base_url: - session = infrastructure.UTMClientSession( - self.config.v1_base_url, auth_adapter, config.timeout_seconds - ) - self.client = V1FlightPlannerClient(session) + self.client = config.to_client(auth_adapter) # Flights injected by this target. self.created_flight_ids: Set[str] = set() diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py index 92595397e9..d61ad2351e 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planners.py @@ -1,6 +1,7 @@ from typing import List, Iterable, Dict, Optional from implicitdict import ImplicitDict +from monitoring.monitorlib.clients.flight_planning.client import FlightPlannerClient from monitoring.uss_qualifier.reports.report import ParticipantID from monitoring.uss_qualifier.resources.definitions import ResourceID @@ -18,6 +19,8 @@ class FlightPlannerSpecification(ImplicitDict): class FlightPlannerResource(Resource[FlightPlannerSpecification]): flight_planner: FlightPlanner + client: FlightPlannerClient + participant_id: ParticipantID def __init__( self, @@ -27,6 +30,8 @@ def __init__( self.flight_planner = FlightPlanner( specification.flight_planner, auth_adapter.adapter ) + self.client = specification.flight_planner.to_client(auth_adapter.adapter) + self.participant_id = specification.flight_planner.participant_id class FlightPlannersSpecification(ImplicitDict): diff --git a/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py b/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py index ee9b7827ae..d73ca3e601 100644 --- a/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py +++ b/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions.py @@ -1,8 +1,10 @@ from enum import Enum -from typing import List, Optional +from typing import List from implicitdict import ImplicitDict -from monitoring.monitorlib.geotemporal import Volume4DTemplate +from monitoring.monitorlib.clients.flight_planning.flight_info_template import ( + FlightInfoTemplate, +) class AcceptanceExpectation(str, Enum): @@ -37,16 +39,8 @@ class FlightCheck(ImplicitDict): description: str """Human-readable test step description to aid in the debugging and traceability.""" - volumes: List[Volume4DTemplate] - """Spatial and temporal definition of the areas the virtual user intends to fly in. - - A service provider is expected to authorizing a flight covering the entire area specified and for any of the entire time specified. - """ - - additional_information: dict - """Any additional information that should be provided to a USS planning the flight. - - Format is agreed upon between test designer and USSs.""" + flight_info: FlightInfoTemplate + """Information about the flight, as a user would provide it to the USS.""" acceptance_expectation: AcceptanceExpectation = AcceptanceExpectation.Irrelevant """Expected outcome when authorizing a flight as described.""" diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md index 4560f36389..2f282ab3e0 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.md @@ -10,12 +10,20 @@ This test acts as a user using a USS's flight planning/authorization interface a [Flight Check Table](../../../resources/interuss/flight_authorization/flight_check_table.py) consisting of a list of Flight Check rows. Each Flight Check row will cause this test to attempt to plan/authorize a flight using the planning/authorization interfaces of each USS under test according to the information in that Flight Check row. This test will then perform checks according to the expected outcomes from those planning/authorization attempts, according to the Flight Check row. +### planner + +[Flight planner](../../../resources/flight_planning/flight_planners.py) providing access to the flight-planning USS under test in this scenario. + ## Flight planning test case ### Dynamic test step The test steps for this test scenario are generated dynamically according to the definitions in the Flight Check Table. The checks for each step are the same and are documented below. +#### Valid planning response check + +The USS under test is expected to implement the InterUSS flight_planning automated testing API and respond to requests accordingly. If the USS does not respond to a flight planning request to this API properly, it will have failed to meet **[interuss.automated_testing.flight_planning.ImplementAPI](../../../requirements/interuss/automated_testing/flight_planning.md)**. + #### Disallowed flight check When the test designer specifies that a particular Flight Check has an expected acceptance of "No", that means attempting to plan/authorize that flight in a USS should result in the request being rejected. Upon this test making this request, if the USS successfully plans/authorizes the flight, this check will fail. @@ -31,3 +39,7 @@ When the test designer specifies that a particular Flight Check's conditions "Mu #### Disallowed conditions check When the test designer specifies that a particular Flight Check's conditions "MustBeAbsent", that means if a flight is successfully planned/authorized, it must NOT be accompanied by any conditions/advisories. If a successfully-planned/authorized flight IS indicated to contain any conditions/advisories, this check will fail. + +#### Successful closure check + +If a flight was successfully planned, then uss_qualifier will emulate a user attempting to close that flight. The flight plan is expected to be Closed following that action. If it is any other value, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../requirements/interuss/automated_testing/flight_planning.md)**. A value of NotPlanned is not acceptable because the flight had previously been planned. diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py index 81f5fb4928..fce56784c4 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py @@ -1,6 +1,19 @@ import arrow +from loguru import logger -from monitoring.monitorlib.geotemporal import resolve_volume4d +from monitoring.monitorlib.clients.flight_planning.client import ( + FlightPlannerClient, + PlanningActivityError, +) +from monitoring.monitorlib.clients.flight_planning.flight_info import ExecutionStyle +from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResult, + FlightPlanStatus, + AdvisoryInclusion, +) +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.resources.flight_planning import FlightPlannerResource from monitoring.uss_qualifier.resources.interuss.flight_authorization.definitions import ( FlightCheckTable, AcceptanceExpectation, @@ -16,10 +29,13 @@ from monitoring.uss_qualifier.scenarios.scenario import TestScenario +# Check names from documentation +_VALID_API_RESPONSE_NAME = "Valid planning response" _ACCEPT_CHECK_NAME = "Allowed flight" _REJECT_CHECK_NAME = "Disallowed flight" _CONDITIONAL_CHECK_NAME = "Required conditions" _UNCONDITIONAL_CHECK_NAME = "Disallowed conditions" +_SUCCESSFUL_CLOSURE_NAME = "Successful closure" def _get_check_by_name( @@ -30,13 +46,18 @@ def _get_check_by_name( class GeneralFlightAuthorization(TestScenario): table: FlightCheckTable + flight_planner: FlightPlannerClient + participant_id: ParticipantID def __init__( self, - table: FlightCheckTableResource, # TODO: Add new flight planner resource + table: FlightCheckTableResource, + planner: FlightPlannerResource, ): super().__init__() self.table = table.table + self.flight_planner = planner.client + self.participant_id = planner.participant_id def run(self): self.begin_test_scenario() @@ -50,7 +71,10 @@ def run(self): def _plan_flights(self): start_time = arrow.utcnow().datetime for row in self.table.rows: - checks = [] + checks = [ + _get_check_by_name(self._current_case.steps[0], name) + for name in (_VALID_API_RESPONSE_NAME, _SUCCESSFUL_CLOSURE_NAME) + ] if row.acceptance_expectation == AcceptanceExpectation.MustBeAccepted: acceptance_check = _get_check_by_name( self._current_case.steps[0], _ACCEPT_CHECK_NAME @@ -103,37 +127,159 @@ def _plan_flights(self): ) self.begin_dynamic_test_step(doc) - concrete_volumes = [resolve_volume4d(v, start_time) for v in row.volumes] + # Attempt planning action + info = row.flight_info.resolve(start_time) + with self.check(_VALID_API_RESPONSE_NAME, [self.participant_id]) as check: + try: + resp = self.flight_planner.try_plan_flight( + info, ExecutionStyle.IfAllowed + ) + except PlanningActivityError as e: + for q in e.queries: + self.record_query(q) + check.record_failed( + summary="Flight planning API request failed", + severity=Severity.High, + details=str(e), + query_timestamps=[ + q.request.initiated_at.datetime for q in e.queries + ], + ) - # TODO: Attempt to plan flight in USSs under test - self.record_note( - "flight_planning", - f"TODO: Attempt to plan flight in USSs where flight plan {row.acceptance_expectation} and conditions {row.conditions_expectation}, from {concrete_volumes[0].time_start} to {concrete_volumes[0].time_end}", - ) + logger.info(f"Recording {len(resp.queries)} queries") + for q in resp.queries: + self.record_query(q) + # Evaluate acceptance result if row.acceptance_expectation == AcceptanceExpectation.MustBeAccepted: - with self.check( - _ACCEPT_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + logger.info("Must be accepted; checking...") + with self.check(_ACCEPT_CHECK_NAME, [self.participant_id]) as check: + if resp.activity_result != PlanningActivityResult.Completed: + check.record_failed( + summary=f"Expected-accepted flight request was {resp.activity_result}", + severity=Severity.Medium, + details=f"The flight was expected to be accepted, but the activity result was indicated as {resp.activity_result}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) + if resp.flight_plan_status not in ( + FlightPlanStatus.Planned, + FlightPlanStatus.OkToFly, + ): + check.record_failed( + summary=f"Expected-accepted flight had {resp.flight_plan_status} flight plan", + severity=Severity.Medium, + details=f"The flight was expected to be accepted, but the flight plan status following the planning action was indicated as {resp.flight_plan_status}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) if row.acceptance_expectation == AcceptanceExpectation.MustBeRejected: - with self.check( - _REJECT_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + logger.info("Must be rejected; checking...") + with self.check(_REJECT_CHECK_NAME, [self.participant_id]) as check: + if resp.activity_result != PlanningActivityResult.Rejected: + check.record_failed( + summary=f"Expected-rejected flight request was {resp.activity_result}", + severity=Severity.Medium, + details=f"The flight was expected to be rejected, but the activity result was indicated as {resp.activity_result}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) + if resp.flight_plan_status != FlightPlanStatus.NotPlanned: + check.record_failed( + summary=f"Expected-accepted flight had {resp.flight_plan_status} flight plan", + severity=Severity.Medium, + details=f"The flight was expected to be rejected, but the flight plan status following the planning action was indicated as {resp.flight_plan_status}", + query_timestamps=[ + q.request.initiated_at.datetime for q in resp.queries + ], + ) - # TODO: Only check conditions expectations if flight planning succeeded - if row.conditions_expectation == ConditionsExpectation.MustBePresent: - with self.check( - _CONDITIONAL_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + # Perform checks only applicable when the planning activity succeeded + if ( + resp.activity_result == PlanningActivityResult.Completed + and resp.flight_plan_status + in (FlightPlanStatus.Planned, FlightPlanStatus.OkToFly) + ): + if row.conditions_expectation == ConditionsExpectation.MustBePresent: + logger.info("Checking conditions must be present...") + with self.check( + _CONDITIONAL_CHECK_NAME, [self.participant_id] + ) as check: + if ( + resp.includes_advisories + != AdvisoryInclusion.AtLeastOneAdvisoryOrCondition + ): + check.record_failed( + summary=f"Missing expected conditions", + severity=Severity.Medium, + details=f"The flight planning activity result was expected to be accompanied by conditions/advisories, but advisory inclusion was {resp.includes_advisories}", + query_timestamps=[ + q.request.initiated_at.datetime + for q in resp.queries + ], + ) - if row.conditions_expectation == ConditionsExpectation.MustBeAbsent: + if row.conditions_expectation == ConditionsExpectation.MustBeAbsent: + logger.info("Checking conditions must be absent...") + with self.check( + _UNCONDITIONAL_CHECK_NAME, [self.participant_id] + ) as check: + if ( + resp.includes_advisories + != AdvisoryInclusion.NoAdvisoriesOrConditions + ): + check.record_failed( + summary=f"Expected-unqualified planning success was qualified by conditions", + severity=Severity.Medium, + details=f"The flight planning activity result was expected to be unqualified (accompanied by no conditions/advisories), but advisory inclusion was {resp.includes_advisories}", + query_timestamps=[ + q.request.initiated_at.datetime + for q in resp.queries + ], + ) + + # Remove flight plan if the activity resulted in a flight plan + if resp.flight_plan_status in ( + FlightPlanStatus.Planned, + FlightPlanStatus.OkToFly, + ): + logger.info("Removing flight...") + with self.check( + _VALID_API_RESPONSE_NAME, [self.participant_id] + ) as check: + try: + del_resp = self.flight_planner.try_end_flight( + resp.flight_id, ExecutionStyle.IfAllowed + ) + except PlanningActivityError as e: + for q in e.queries: + self.record_query(q) + check.record_failed( + summary="Flight planning API delete request failed", + severity=Severity.High, + details=str(e), + query_timestamps=[ + q.request.initiated_at.datetime for q in e.queries + ], + ) + for q in del_resp.queries: + self.record_query(q) with self.check( - _UNCONDITIONAL_CHECK_NAME, [] - ) as check: # TODO: Add participant_id - pass # TODO: check USS planning results + _SUCCESSFUL_CLOSURE_NAME, [self.participant_id] + ) as check: + if del_resp.flight_plan_status != FlightPlanStatus.Closed: + check.record_failed( + summary="Could not close flight plan successfully", + severity=Severity.High, + details=f"Expected flight plan status to be Closed after request to end flight, but status was instead {del_resp.flight_plan_status}", + query_timestamps=[ + q.request.initiated_at.datetime + for q in del_resp.queries + ], + ) self.end_test_step() diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json new file mode 100644 index 0000000000..24ad42f3dc --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json @@ -0,0 +1,18 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/ASTMF354821OpIntentInformation.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Information provided about a flight plan that is necessary for ASTM F3548-21.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info.ASTMF354821OpIntentInformation, as defined in monitoring/monitorlib/clients/flight_planning/flight_info.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "priority": { + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json new file mode 100644 index 0000000000..2d8a22cef9 --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json @@ -0,0 +1,98 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/FlightAuthorisationData.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "The details of a UAS flight authorization request, as received from the user.\n\nNote that a full description of a flight authorisation must include mandatory information required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664 for an UAS flight authorisation request. Reference: https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:32021R0664&from=EN#d1e32-178-1\n\nmonitoring.monitorlib.clients.flight_planning.flight_info.FlightAuthorisationData, as defined in monitoring/monitorlib/clients/flight_planning/flight_info.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "connectivity_methods": { + "description": "Connectivity methods. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 7.", + "items": { + "type": "string" + }, + "type": "array" + }, + "emergency_procedure_url": { + "description": "The URL at which the applicable emergency procedure in case of a loss of command and control link may be retrieved. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 9.", + "type": "string" + }, + "endurance_minutes": { + "description": "Endurance of the UAS. This is expressed in minutes. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 8.", + "type": "integer" + }, + "identification_technologies": { + "description": "Technology used to identify the UAS. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 6.", + "items": { + "type": "string" + }, + "type": "array" + }, + "operation_category": { + "description": "Category of UAS operation (\u2018open\u2019, \u2018specific\u2019, \u2018certified\u2019) as defined in COMMISSION DELEGATED REGULATION (EU) 2019/945. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.", + "enum": [ + "Unknown", + "Open", + "Specific", + "Certified" + ], + "type": "string" + }, + "operation_mode": { + "enum": [ + "Undeclared", + "Vlos", + "Bvlos" + ], + "type": "string" + }, + "operator_id": { + "description": "Registration number of the UAS operator.\nThe format is defined in EASA Easy Access Rules for Unmanned Aircraft Systems GM1 to AMC1\nArticle 14(6) Registration of UAS operators and \u2018certified\u2019 UAS.\nRequired by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 10.", + "type": "string" + }, + "uas_class": { + "enum": [ + "Other", + "C0", + "C1", + "C2", + "C3", + "C4", + "C5", + "C6" + ], + "type": "string" + }, + "uas_id": { + "description": "When applicable, the registration number of the unmanned aircraft.\nThis is expressed using the nationality and registration mark of the unmanned aircraft in\nline with ICAO Annex 7.\nSpecified by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 10.", + "type": [ + "string", + "null" + ] + }, + "uas_serial_number": { + "description": "Unique serial number of the unmanned aircraft or, if the unmanned aircraft is privately built, the unique serial number of the add-on. This is expressed in the ANSI/CTA-2063 Physical Serial Number format. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 1.", + "type": "string" + }, + "uas_type_certificate": { + "description": "Provisional field. Not applicable as of September 2021. Required only if `uas_class` is set to `other` by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "connectivity_methods", + "emergency_procedure_url", + "endurance_minutes", + "identification_technologies", + "operation_category", + "operation_mode", + "operator_id", + "uas_class", + "uas_serial_number" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json new file mode 100644 index 0000000000..99a396c9fa --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json @@ -0,0 +1,102 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info/RPAS26FlightDetails.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Information about a flight necessary to plan successfully using the RPAS Platform Operating Rules version 2.6.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info.RPAS26FlightDetails, as defined in monitoring/monitorlib/clients/flight_planning/flight_info.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "aircraft_type": { + "description": "Type of vehicle being used as per ASTM F3411-22a.", + "enum": [ + "NotDeclared", + "Aeroplane", + "Helicopter", + "Gyroplane", + "HybridLift", + "Ornithopter", + "Glider", + "Kite", + "FreeBalloon", + "CaptiveBalloon", + "Airship", + "FreeFallOrParachute", + "Rocket", + "TetheredPoweredAircraft", + "GroundObstacle", + "Other" + ], + "type": [ + "string", + "null" + ] + }, + "flight_profile": { + "description": "Type of flight profile.", + "enum": [ + "AutomatedGrid", + "AutomatedWaypoint", + "Manual" + ], + "type": [ + "string", + "null" + ] + }, + "operator_number": { + "description": "Operator number.", + "type": [ + "string", + "null" + ] + }, + "operator_type": { + "description": "The type of operator.", + "enum": [ + "Recreational", + "CommercialExcluded", + "ReOC" + ], + "type": [ + "string", + "null" + ] + }, + "pilot_license_number": { + "description": "License number for the pilot.", + "type": [ + "string", + "null" + ] + }, + "pilot_phone_number": { + "description": "Contact phone number for the pilot.", + "type": [ + "string", + "null" + ] + }, + "uas_registration_numbers": { + "description": "The list of UAS/drone registration numbers that will be operated during the operation.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "uas_serial_numbers": { + "description": "The list of UAS/drone serial numbers that will be operated during the operation.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json new file mode 100644 index 0000000000..728cb5af9d --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json @@ -0,0 +1,41 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/BasicFlightPlanInformationTemplate.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Template to provide (at runtime) basic information about a flight plan that an operator and/or UAS can be expected to provide in most flight planning scenarios.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info_template.BasicFlightPlanInformationTemplate, as defined in monitoring/monitorlib/clients/flight_planning/flight_info_template.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "area": { + "description": "User intends to or may fly anywhere in this entire area.", + "items": { + "$ref": "../../../geotemporal/Volume4DTemplate.json" + }, + "type": "array" + }, + "uas_state": { + "description": "State of the user's UAS associated with this flight plan.", + "enum": [ + "Nominal", + "OffNominal", + "Contingent" + ], + "type": "string" + }, + "usage_state": { + "description": "User's current usage of the airspace specified in the flight plan.", + "enum": [ + "Planned", + "InUse" + ], + "type": "string" + } + }, + "required": [ + "area", + "uas_state", + "usage_state" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json new file mode 100644 index 0000000000..c44de2b563 --- /dev/null +++ b/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json @@ -0,0 +1,55 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Template to provide (at runtime) details of user's intent to create or modify a flight plan.\n\nmonitoring.monitorlib.clients.flight_planning.flight_info_template.FlightInfoTemplate, as defined in monitoring/monitorlib/clients/flight_planning/flight_info_template.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "additional_information": { + "description": "Any information relevant to a particular jurisdiction or use case not described in the standard schema. The keys and values must be agreed upon between the test designers and USSs under test.", + "type": [ + "object", + "null" + ] + }, + "astm_f3548_21": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../flight_info/ASTMF354821OpIntentInformation.json" + } + ] + }, + "basic_information": { + "$ref": "BasicFlightPlanInformationTemplate.json" + }, + "rpas_operating_rules_2_6": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../flight_info/RPAS26FlightDetails.json" + } + ] + }, + "uspace_flight_authorisation": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../flight_info/FlightAuthorisationData.json" + } + ] + } + }, + "required": [ + "basic_information" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/monitorlib/fetch/Query.json b/schemas/monitoring/monitorlib/fetch/Query.json index 0255097971..d69788f3cf 100644 --- a/schemas/monitoring/monitorlib/fetch/Query.json +++ b/schemas/monitoring/monitorlib/fetch/Query.json @@ -45,7 +45,11 @@ "astm.f3548.v21.uss.getConstraintDetails", "astm.f3548.v21.uss.notifyConstraintDetailsChanged", "astm.f3548.v21.uss.makeUssReport", - "interuss.automated_testing.versioning.GetVersion" + "interuss.automated_testing.versioning.GetVersion", + "interuss.automated_testing.flight_planning.v1.GetStatus", + "interuss.automated_testing.flight_planning.v1.ClearArea", + "interuss.automated_testing.flight_planning.v1.UpsertFlightPlan", + "interuss.automated_testing.flight_planning.v1.DeleteFlightPlan" ], "type": [ "string", diff --git a/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json b/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json index 7ee3137a1d..7c40f148cb 100644 --- a/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json +++ b/schemas/monitoring/uss_qualifier/resources/interuss/flight_authorization/definitions/FlightCheck.json @@ -16,10 +16,6 @@ ], "type": "string" }, - "additional_information": { - "description": "Any additional information that should be provided to a USS planning the flight.\n\nFormat is agreed upon between test designer and USSs.", - "type": "object" - }, "conditions_expectation": { "description": "Expected conditions/advisories produced when authorizing a flight as described.", "enum": [ @@ -37,27 +33,23 @@ "description": "Unique (within table) test step/row identifier.", "type": "string" }, + "flight_info": { + "$ref": "../../../../../monitorlib/clients/flight_planning/flight_info_template/FlightInfoTemplate.json", + "description": "Information about the flight, as a user would provide it to the USS." + }, "requirement_ids": { "description": "Jurisdictional identifiers of the requirements this test step is evaluating.", "items": { "type": "string" }, "type": "array" - }, - "volumes": { - "description": "Spatial and temporal definition of the areas the virtual user intends to fly in.\n\nA service provider is expected to authorizing a flight covering the entire area specified and for any of the entire time specified.", - "items": { - "$ref": "../../../../../monitorlib/geotemporal/Volume4DTemplate.json" - }, - "type": "array" } }, "required": [ - "additional_information", "description", "flight_check_id", - "requirement_ids", - "volumes" + "flight_info", + "requirement_ids" ], "type": "object" } \ No newline at end of file