From a831feaee7cdb3d788d2352bde5962a58caaa2e3 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Wed, 11 Oct 2023 13:18:59 -0700 Subject: [PATCH 1/3] [uss_qualifier] Add sequence view artifact (#240) * Add sequence view artifact * Fix notes in failed test step --- monitoring/mock_uss/templates/tracer/log.html | 4 +- monitoring/monitorlib/fetch/__init__.py | 5 + .../monitorlib/html/templates/explorer.html | 14 +- .../configurations/configuration.py | 10 +- .../configurations/dev/uspace.yaml | 2 + monitoring/uss_qualifier/main.py | 6 + monitoring/uss_qualifier/reports/report.py | 3 + .../uss_qualifier/reports/sequence_view.py | 491 ++++++++++++++++++ .../reports/templates/report.html | 4 +- .../templates/sequence_view/overview.html | 104 ++++ .../templates/sequence_view/scenario.html | 188 +++++++ .../uss_qualifier/scenarios/scenario.py | 3 +- .../configuration/ArtifactsConfiguration.json | 11 + .../ReportHTMLConfiguration.json | 16 +- .../SequenceViewConfiguration.json | 19 + .../reports/report/PassedCheck.json | 36 +- 16 files changed, 881 insertions(+), 35 deletions(-) create mode 100644 monitoring/uss_qualifier/reports/sequence_view.py create mode 100644 monitoring/uss_qualifier/reports/templates/sequence_view/overview.html create mode 100644 monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html create mode 100644 schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json diff --git a/monitoring/mock_uss/templates/tracer/log.html b/monitoring/mock_uss/templates/tracer/log.html index 5fac93169e..dd0158b975 100644 --- a/monitoring/mock_uss/templates/tracer/log.html +++ b/monitoring/mock_uss/templates/tracer/log.html @@ -3,6 +3,6 @@ {% block content %} {{ explorer_header() }} - {{ explorer_content(log) }} - {{ explorer_footer() }} + {{ explorer_content("top_node", log) }} + {{ explorer_footer(["top_node"]) }} {% endblock %} diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index f9b1cadcdf..b99816b69a 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -6,6 +6,7 @@ from typing import Dict, Optional, List from enum import Enum +from urllib.parse import urlparse import flask from loguru import logger @@ -56,6 +57,10 @@ def timestamp(self) -> datetime.datetime: "RequestDescription missing both initiated_at and received_at" ) + @property + def url_hostname(self) -> str: + return urlparse(self.url).hostname + yaml.add_representer(RequestDescription, Representer.represent_dict) diff --git a/monitoring/monitorlib/html/templates/explorer.html b/monitoring/monitorlib/html/templates/explorer.html index 9b6b1ee03c..af2e2d2bfa 100644 --- a/monitoring/monitorlib/html/templates/explorer.html +++ b/monitoring/monitorlib/html/templates/explorer.html @@ -1,7 +1,7 @@ {# Renders the provided `obj` dict as interactive HTML #} {# Content of explorer_header() should be added to the header of the page #} -{# Content of explorer_content(obj) represents the `obj` dict as interactive HTML content #} -{# Content of explorer_footer() should be added to the page such that it is loaded after draw_node #} +{# Content of explorer_content(div_id, obj) represents the `obj` dict as interactive HTML content #} +{# Content of explorer_footer(div_ids) should be added to the page such that it is loaded after explorer_content/draw_node #} {% macro collapseable(v) %}{% if v is mapping or (v is iterable and v is not string) %}collapseable{% else %}not_collapseable{% endif %}{% endmacro %} @@ -66,13 +66,13 @@ {% endmacro %} -{% macro explorer_content(obj) %} -
+{% macro explorer_content(div_id, obj) %} +
{{ draw_node(obj) }}
{% endmacro %} -{% macro explorer_footer() %} +{% macro explorer_footer(div_ids) %} {% endmacro %} diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index 398d4374fc..e653164609 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -50,9 +50,14 @@ class TestedRequirementsConfiguration(ImplicitDict): """If a requirement collection is specified for a participant, only the requirements in the specified collection will be listed on that participant's report.""" +class SequenceViewConfiguration(ImplicitDict): + output_path: str + """Path of a folder into which report HTML files should be written""" + + class ReportHTMLConfiguration(ImplicitDict): html_path: str - """Path of HTML file to contain an HTML rendering of the test report""" + """Path of HTML file to contain an HTML rendering of the raw test report object""" class TemplatedReportInjectedConfiguration(ImplicitDict): @@ -102,6 +107,9 @@ class ArtifactsConfiguration(ImplicitDict): tested_requirements: Optional[TestedRequirementsConfiguration] = None """If specified, configuration describing a desired report summarizing all tested requirements for each participant""" + sequence_view: Optional[SequenceViewConfiguration] = None + """If specified, configuration describing a desired report describing the sequence of events that occurred during the test""" + class USSQualifierConfigurationV1(ImplicitDict): test_run: Optional[TestConfiguration] = None diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index 45f32f6913..e7b54c7711 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -44,3 +44,5 @@ v1: participant_requirements: uss1: uspace uss2: uspace + sequence_view: + output_path: output/sequence_uspace diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py index 240435f433..a8e9afd2a7 100644 --- a/monitoring/uss_qualifier/main.py +++ b/monitoring/uss_qualifier/main.py @@ -19,6 +19,7 @@ ) from monitoring.uss_qualifier.fileio import load_dict_with_references from monitoring.uss_qualifier.reports.documents import make_report_html +from monitoring.uss_qualifier.reports.sequence_view import generate_sequence_view from monitoring.uss_qualifier.reports.tested_requirements import ( generate_tested_requirements, ) @@ -204,6 +205,11 @@ def main() -> int: logger.info(f"Writing tested requirements view to {path}") generate_tested_requirements(report, config.artifacts.tested_requirements) + if config.artifacts.sequence_view: + path = config.artifacts.sequence_view.output_path + logger.info(f"Writing sequence view to {path}") + generate_sequence_view(report, config.artifacts.sequence_view) + return os.EX_OK diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index 818f274119..b0e1a2bef4 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -59,6 +59,9 @@ class PassedCheck(ImplicitDict): name: str """Name of the check that passed""" + timestamp: StringBasedDateTime + """Time the issue was discovered""" + requirements: List[RequirementID] """Requirements that would not have been met if this check had failed""" diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py new file mode 100644 index 0000000000..40c19c774d --- /dev/null +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -0,0 +1,491 @@ +from __future__ import annotations +import os +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import List, Dict, Optional, Iterator + +from implicitdict import ImplicitDict + +from monitoring.monitorlib.fetch import Query +from monitoring.uss_qualifier.configurations.configuration import ( + ParticipantID, + SequenceViewConfiguration, +) +from monitoring.uss_qualifier.reports import jinja_env +from monitoring.uss_qualifier.reports.report import ( + TestRunReport, + TestSuiteActionReport, + TestScenarioReport, + PassedCheck, + FailedCheck, +) +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName + + +class NoteEvent(ImplicitDict): + key: str + message: str + timestamp: datetime + + +class EventType(str, Enum): + PassedCheck = "PassedCheck" + FailedCheck = "FailedCheck" + Query = "Query" + Note = "Note" + + +class Event(ImplicitDict): + event_index: int = 0 + passed_check: Optional[PassedCheck] = None + failed_check: Optional[FailedCheck] = None + query: Optional[Query] = None + note: Optional[NoteEvent] = None + + @property + def type(self) -> EventType: + if self.passed_check: + return EventType.PassedCheck + elif self.failed_check: + return EventType.FailedCheck + elif self.query: + return EventType.Query + elif self.note: + return EventType.Note + else: + raise ValueError("Invalid Event type") + + @property + def timestamp(self) -> datetime: + if self.passed_check: + return self.passed_check.timestamp.datetime + elif self.failed_check: + return self.failed_check.timestamp.datetime + elif self.query: + return self.query.request.timestamp + elif self.note: + return self.note.timestamp + else: + raise ValueError("Invalid Event type") + + +class TestedStep(ImplicitDict): + name: str + url: str + events: List[Event] + + @property + def rows(self) -> int: + return len(self.events) + + +class TestedCase(ImplicitDict): + name: str + url: str + steps: List[TestedStep] + + @property + def rows(self) -> int: + return sum(s.rows for s in self.steps) + + +class EpochType(str, Enum): + Case = "Case" + Events = "Events" + + +class Epoch(ImplicitDict): + case: Optional[TestedCase] = None + events: Optional[List[Event]] = None + + @property + def type(self) -> EpochType: + if self.case: + return EpochType.Case + elif self.events: + return EpochType.Events + else: + raise ValueError("Invalid Epoch did not specify case or events") + + @property + def rows(self) -> int: + if self.case: + return self.case.rows + elif self.events: + return len(self.events) + else: + raise ValueError("Invalid Epoch did not specify case or events") + + +@dataclass +class TestedParticipant(object): + has_failures: bool + + +class TestedScenario(ImplicitDict): + type: TestScenarioTypeName + name: str + url: str + scenario_index: int + epochs: List[Epoch] + participants: Dict[ParticipantID, TestedParticipant] + + @property + def rows(self) -> int: + return sum(c.rows for c in self.epochs) + + +class ActionNodeType(str, Enum): + Scenario = "Scenario" + Suite = "Suite" + ActionGenerator = "ActionGenerator" + + +class ActionNode(ImplicitDict): + name: str + node_type: ActionNodeType + children: List[ActionNode] + scenario: Optional[TestedScenario] = None + + @property + def rows(self) -> int: + return sum(c.rows for c in self.children) if self.children else 1 + + @property + def cols(self) -> int: + return 1 + max(c.cols for c in self.children) if self.children else 1 + + +@dataclass +class Indexer(object): + scenario_index: int = 1 + + +@dataclass +class SuiteCell(object): + node: Optional[ActionNode] + first_row: bool + rowspan: int = 1 + colspan: int = 1 + + +@dataclass +class OverviewRow(object): + suite_cells: List[SuiteCell] + scenario_node: ActionNode + filled: bool = False + + +def _compute_tested_scenario( + report: TestScenarioReport, indexer: Indexer +) -> TestedScenario: + epochs = [] + event_index = 1 + + def append_notes(new_notes): + nonlocal event_index + events = [] + for k, v in new_notes.items(): + events.append( + Event( + note=NoteEvent( + key=k, message=v.message, timestamp=v.timestamp.datetime + ), + event_index=event_index, + ) + ) + event_index += 1 + events.sort(key=lambda e: e.timestamp) + epochs.append(Epoch(events=events)) + + # Add any notes that occurred before the first test step + if "notes" in report and report.notes: + if len(report.cases) >= 1 and len(report.cases[0].steps) >= 1: + first_step_start = report.cases[0].steps[0].start_time.datetime + pre_notes = { + k: v + for k, v in report.notes.items() + if v.timestamp.datetime < first_step_start + } + else: + pre_notes = report.notes + if pre_notes: + append_notes(pre_notes) + + scenario_participants: Dict[ParticipantID, TestedParticipant] = {} + + latest_step_time = None + for case in report.cases: + steps = [] + last_step = None + for step in case.steps: + if "notes" in report and report.notes: + # Add events (notes) that happened in between the previous step and this one + if last_step is not None: + inter_notes = { + k: v + for k, v in report.notes.items() + if last_step.end_time.datetime + < v.timestamp.datetime + < step.start_time.datetime + } + if inter_notes: + append_notes(inter_notes) + else: + last_step = step + + # Enumerate the events of this step + events = [] + for passed_check in step.passed_checks: + events.append(Event(passed_check=passed_check)) + for pid in passed_check.participants: + p = scenario_participants.get( + pid, TestedParticipant(has_failures=False) + ) + scenario_participants[pid] = p + for failed_check in step.failed_checks: + events.append(Event(failed_check=failed_check)) + for pid in failed_check.participants: + p = scenario_participants.get( + pid, TestedParticipant(has_failures=True) + ) + p.has_failures = True + scenario_participants[pid] = p + if "queries" in step and step.queries: + for query in step.queries: + events.append(Event(query=query)) + if "server_id" in query and query.server_id: + p = scenario_participants.get( + query.server_id, TestedParticipant(has_failures=False) + ) + scenario_participants[query.server_id] = p + if "notes" in report and report.notes: + for key, note in report.notes.items(): + if step.start_time.datetime <= note.timestamp.datetime: + if ( + "end_time" not in step + or note.timestamp.datetime <= step.end_time.datetime + ): + events.append( + Event( + note=NoteEvent( + key=key, + message=note.message, + timestamp=note.timestamp.datetime, + ) + ) + ) + + # Sort this step's events by time + events.sort(key=lambda e: e.timestamp) + + # Label this step's events with event_index + for e in events: + e.event_index = event_index + event_index += 1 + + # Look for the latest time something happened + for e in events: + if latest_step_time is None or e.timestamp > latest_step_time: + latest_step_time = e.timestamp + if "end_time" in step and step.end_time: + if ( + latest_step_time is None + or step.end_time.datetime > latest_step_time + ): + latest_step_time = step.end_time.datetime + + # Add this step + steps.append( + TestedStep( + name=step.name, + url=step.documentation_url, + events=events, + ) + ) + epochs.append( + Epoch( + case=TestedCase(name=case.name, url=case.documentation_url, steps=steps) + ) + ) + + # Add any notes that occurred after the last test step + if "notes" in report and report.notes: + if len(report.cases) >= 1 and len(report.cases[0].steps) >= 1: + post_notes = { + k: v + for k, v in report.notes.items() + if v.timestamp.datetime > latest_step_time + } + else: + post_notes = {} + if post_notes: + append_notes(post_notes) + + scenario = TestedScenario( + type=report.scenario_type, + name=report.name, + url=report.documentation_url, + epochs=epochs, + scenario_index=indexer.scenario_index, + participants=scenario_participants, + ) + indexer.scenario_index += 1 + return scenario + + +def _compute_action_node(report: TestSuiteActionReport, indexer: Indexer) -> ActionNode: + ( + is_test_suite, + is_test_scenario, + is_action_generator, + ) = report.get_applicable_report() + if is_test_scenario: + return ActionNode( + name=report.test_scenario.name, + node_type=ActionNodeType.Scenario, + children=[], + scenario=_compute_tested_scenario(report.test_scenario, indexer), + ) + elif is_test_suite: + return ActionNode( + name=report.test_suite.name, + node_type=ActionNodeType.Suite, + children=[ + _compute_action_node(a, indexer) for a in report.test_suite.actions + ], + ) + elif is_action_generator: + return ActionNode( + name=report.action_generator.generator_type, + node_type=ActionNodeType.ActionGenerator, + children=[ + _compute_action_node(a, indexer) + for a in report.action_generator.actions + ], + ) + else: + raise ValueError( + "Invalid TestSuiteActionReport; doesn't specify scenario, suite, or action generator" + ) + + +def _compute_overview_rows(node: ActionNode) -> Iterator[OverviewRow]: + if node.node_type == ActionNodeType.Scenario: + yield OverviewRow(suite_cells=[], scenario_node=node) + else: + first_row = True + for child in node.children: + for row in _compute_overview_rows(child): + yield OverviewRow( + suite_cells=[SuiteCell(node=node, first_row=first_row)] + + row.suite_cells, + scenario_node=row.scenario_node, + ) + first_row = False + + +def _align_overview_rows(rows: List[OverviewRow]) -> None: + max_suite_cols = max(len(r.suite_cells) for r in rows) + to_fill = 0 + for row in rows: + if to_fill > 0: + row.filled = True + to_fill -= 1 + elif len(row.suite_cells) < max_suite_cols: + if row.suite_cells[-1].first_row and all( + c.node_type == ActionNodeType.Scenario + for c in row.suite_cells[-1].node.children + ): + row.suite_cells[-1].colspan += max_suite_cols - len(row.suite_cells) + row.filled = True + to_fill = row.suite_cells[-1].node.rows - 1 + + r0 = 0 + while r0 < len(rows): + if len(rows[r0].suite_cells) < max_suite_cols and not rows[r0].filled: + r1 = r0 + 1 + while r1 < len(rows): + if ( + len(rows[r1].suite_cells) != len(rows[r0].suite_cells) + or rows[r1].suite_cells[-1].node != rows[r0].suite_cells[-1].node + ): + break + r1 += 1 + rows[r0].suite_cells.append( + SuiteCell( + node=None, + first_row=True, + rowspan=r1 - r0, + colspan=max_suite_cols - len(rows[r0].suite_cells), + ) + ) + rows[r0].filled = True + r0 = r1 + else: + r0 += 1 + + +def _enumerate_all_participants(node: ActionNode) -> List[ParticipantID]: + if node.node_type == ActionNodeType.Scenario: + return list(node.scenario.participants) + else: + result = set() + for child in node.children: + for p in _enumerate_all_participants(child): + result.add(p) + return list(result) + + +def _generate_scenario_pages( + node: ActionNode, config: SequenceViewConfiguration +) -> None: + if node.node_type == ActionNodeType.Scenario: + all_participants = list(node.scenario.participants) + all_participants.sort() + scenario_file = os.path.join( + config.output_path, f"s{node.scenario.scenario_index}.html" + ) + template = jinja_env.get_template("sequence_view/scenario.html") + with open(scenario_file, "w") as f: + f.write( + template.render( + test_scenario=node.scenario, + all_participants=all_participants, + EpochType=EpochType, + EventType=EventType, + len=len, + str=str, + ) + ) + else: + for child in node.children: + _generate_scenario_pages(child, config) + + +def generate_sequence_view( + report: TestRunReport, config: SequenceViewConfiguration +) -> None: + node = _compute_action_node(report.report, Indexer()) + + os.makedirs(config.output_path, exist_ok=True) + _generate_scenario_pages(node, config) + + overview_rows = list(_compute_overview_rows(node)) + _align_overview_rows(overview_rows) + max_suite_cols = max(len(r.suite_cells) for r in overview_rows) + all_participants = _enumerate_all_participants(node) + all_participants.sort() + overview_file = os.path.join(config.output_path, "index.html") + template = jinja_env.get_template("sequence_view/overview.html") + with open(overview_file, "w") as f: + f.write( + template.render( + overview_rows=overview_rows, + max_suite_cols=max_suite_cols, + all_participants=all_participants, + ActionNodeType=ActionNodeType, + len=len, + ) + ) diff --git a/monitoring/uss_qualifier/reports/templates/report.html b/monitoring/uss_qualifier/reports/templates/report.html index 18f2ef083f..03b0dad560 100644 --- a/monitoring/uss_qualifier/reports/templates/report.html +++ b/monitoring/uss_qualifier/reports/templates/report.html @@ -8,8 +8,8 @@
{{ explorer_header() }} - {{ explorer_content(report) }} - {{ explorer_footer() }} + {{ explorer_content("top_node", report) }} + {{ explorer_footer(["top_node"]) }}
diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html new file mode 100644 index 0000000000..963cda2fc5 --- /dev/null +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html @@ -0,0 +1,104 @@ + + + + + +
+ + + {% if max_suite_cols > 0 %} + + {% endif %} + + {% for participant_id in all_participants %} + + {% endfor %} + + {% for row in overview_rows %} + + {% for suite_cell in row.suite_cells %} + {% if suite_cell.first_row %} + {% if suite_cell.node != None %} + + {% else %} + + {% endif %} + {% endif %} + {% endfor %} + + {% for participant_id in all_participants %} + {% if participant_id in row.scenario_node.scenario.participants %} + {% if row.scenario_node.scenario.participants[participant_id].has_failures %} + + {% else %} + + {% endif %} + {% else %} + + {% endif %} + {% endfor %} + + {% endfor %} +
Suite / action generatorScenario{{ participant_id }}
{{ suite_cell.node.name }} + {{ row.scenario_node.scenario.name }} + ❌✅
+
+ + diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html new file mode 100644 index 0000000000..c2fafc2f73 --- /dev/null +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html @@ -0,0 +1,188 @@ + +{% from "explorer.html" import explorer_header, explorer_content, explorer_footer %} + + + + s{{ test_scenario.scenario_index }} - {{ test_scenario.name }} + + {{ explorer_header() }} + + +{% set collapsible = namespace(queries=[]) %} +
+ {% if test_scenario.url %} +

{{ test_scenario.name }}

+ {% else %} +

{{ test_scenario.name }}

+ {% endif %} +

{{ test_scenario.type }}

+ + + + + + {% for participant_id in all_participants %} + + {% endfor %} + + + {% set first_row = namespace(epoch=True, step=True) %} + {% for epoch in test_scenario.epochs %} + {% set first_row.epoch = True %} + {% if epoch.type == EpochType.Case %} + {% for test_step in epoch.case.steps %} + {% set first_row.step = True %} + {% for event in test_step.events %} + + {% if first_row.epoch %} + + {% endif %} + {% if first_row.step %} + + {% endif %} + + {% if event.type == EventType.PassedCheck %} + + + {% for participant_id in all_participants %} + {% if participant_id in event.passed_check.participants %} + + {% else %} + + {% endif %} + {% endfor %} + {% elif event.type == EventType.FailedCheck %} + + + {% for participant_id in all_participants %} + {% if participant_id in event.failed_check.participants %} + + {% else %} + + {% endif %} + {% endfor %} + {% elif event.type == EventType.Query %} + + + {% for participant_id in all_participants %} + {% if participant_id == event.query.get("server_id", None) %} + + {% else %} + + {% endif %} + {% endfor %} + {% elif event.type == EventType.Note %} + + + {% else %} + + {% endif %} + + {% set first_row.epoch = False %} + {% set first_row.step = False %} + {% endfor %} + {% endfor %} + {% elif epoch.type == EpochType.Events %} + {% for event in epoch.events %} + + {% if first_row.epoch %} + + {% endif %} + + + + + {% set first_row.epoch = False %} + {% endfor %} + {% endif %} + {% endfor %} +
CaseStepEvent{{ participant_id }}
+ {% if epoch.case.url %} + {{ epoch.case.name }} + {% else %} + {{ epoch.case.name }} + {% endif %} + + {% if test_step.url %} + {{ test_step.name }} + {% else %} + {{ test_step.name }} + {% endif %} + {{ event.event_index }}✅ + {{ event.passed_check.name }} + ✅❌ + {% if event.failed_check.documentation_url %} + {{ event.failed_check.name }} + {% else %} + {{ event.failed_check.name }} + {% endif %} + ❌🌐 + {{ event.query.request.method }} {{ event.query.request.url_hostname }} + {% set query_id = "e" + str(event.event_index) + "query" %} + {{ explorer_content(query_id, event.query) }} + {% set collapsible.queries = collapsible.queries + [query_id] %} + 🌐📓 + {{ event.note.key }}: {{ event.note.value }} + ???Render error: unknown EventType '{{ event.type }}'
{{ event.event_index }}📓 + {{ event.note.key }}: {{ event.note.message }} +
+
+{{ explorer_footer(collapsible.queries) }} + + diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index b04f702d0f..2ee5aec44b 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -122,7 +122,7 @@ def record_failed( kwargs = { "name": self._documentation.name, "documentation_url": self._documentation.url, - "timestamp": StringBasedDateTime(datetime.utcnow()), + "timestamp": StringBasedDateTime(arrow.utcnow()), "summary": summary, "details": details, "requirements": requirements, @@ -157,6 +157,7 @@ def record_passed( passed_check = PassedCheck( name=self._documentation.name, + timestamp=StringBasedDateTime(arrow.utcnow()), participants=participants, requirements=requirements, ) diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json index 67fb992e24..65a548b622 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ArtifactsConfiguration.json @@ -44,6 +44,17 @@ } ] }, + "sequence_view": { + "description": "If specified, configuration describing a desired report describing the sequence of events that occurred during the test", + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "SequenceViewConfiguration.json" + } + ] + }, "templated_reports": { "description": "List of report templates to be rendered", "items": { diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json index 4ad3d1fafc..1877035482 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json @@ -1,19 +1,19 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.configurations.configuration.ReportHTMLConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, "html_path": { - "type": "string", - "description": "Path of HTML file to contain an HTML rendering of the test report" + "description": "Path of HTML file to contain an HTML rendering of the raw test report object", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/ReportHTMLConfiguration.json", - "description": "monitoring.uss_qualifier.configurations.configuration.ReportHTMLConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", "required": [ "html_path" - ] + ], + "type": "object" } \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json new file mode 100644 index 0000000000..f1428d02a2 --- /dev/null +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json @@ -0,0 +1,19 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/configurations/configuration/SequenceViewConfiguration.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.uss_qualifier.configurations.configuration.SequenceViewConfiguration, as defined in monitoring/uss_qualifier/configurations/configuration.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "output_path": { + "description": "Path of a folder into which report HTML files should be written", + "type": "string" + } + }, + "required": [ + "output_path" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json b/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json index 30ec39ed0a..efa921727c 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json +++ b/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json @@ -1,35 +1,41 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.uss_qualifier.reports.report.PassedCheck, as defined in monitoring/uss_qualifier/reports/report.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "name": { + "description": "Name of the check that passed", + "type": "string" }, "participants": { - "type": "array", + "description": "Participants that may not have met the relevant requirements if this check had failed", "items": { "type": "string" }, - "description": "Participants that may not have met the relevant requirements if this check had failed" - }, - "name": { - "type": "string", - "description": "Name of the check that passed" + "type": "array" }, "requirements": { - "type": "array", + "description": "Requirements that would not have been met if this check had failed", "items": { "type": "string" }, - "description": "Requirements that would not have been met if this check had failed" + "type": "array" + }, + "timestamp": { + "description": "Time the issue was discovered", + "format": "date-time", + "type": "string" } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/PassedCheck.json", - "description": "monitoring.uss_qualifier.reports.report.PassedCheck, as defined in monitoring/uss_qualifier/reports/report.py", "required": [ "name", "participants", - "requirements" - ] + "requirements", + "timestamp" + ], + "type": "object" } \ No newline at end of file From a76b519d880bb4d3945874aa25c6317f4a56f85c Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Wed, 11 Oct 2023 14:14:27 -0700 Subject: [PATCH 2/3] [mock_uss] Configure mock_uss locality at runtime (#239) Add configurable locality for mock_uss --- monitoring/mock_uss/auth.py | 2 + monitoring/mock_uss/config.py | 5 +- .../dynamic_configuration/__init__.py | 0 .../dynamic_configuration/configuration.py | 29 ++++ .../mock_uss/dynamic_configuration/routes.py | 48 ++++++ .../mock_uss/interaction_logging/logger.py | 2 +- monitoring/mock_uss/routes.py | 3 + monitoring/mock_uss/scdsc/routes_injection.py | 9 +- .../monitorlib/clients/mock_uss/__init__.py | 0 .../monitorlib/clients/mock_uss/locality.py | 15 ++ monitoring/monitorlib/infrastructure.py | 7 +- monitoring/monitorlib/locality.py | 37 +++-- .../action_generators/interuss/__init__.py | 0 .../interuss/mock_uss/__init__.py | 1 + .../interuss/mock_uss/with_locality.py | 148 ++++++++++++++++++ .../dev/faa/uft/local_message_signing.yaml | 2 +- .../dev/library/environment.yaml | 13 ++ .../configurations/dev/library/resources.yaml | 9 ++ .../configurations/dev/uspace.yaml | 29 +++- monitoring/uss_qualifier/reports/report.py | 23 +-- .../interuss/mock_uss/hosted_instance.md | 11 ++ .../resources/interuss/__init__.py | 1 - .../resources/interuss/mock_uss/__init__.py | 0 .../{mock_uss.py => mock_uss/client.py} | 52 ++++++ .../resources/interuss/mock_uss/locality.py | 18 +++ .../faa/uft/message_signing_finalize.py | 2 +- .../faa/uft/message_signing_start.py | 2 +- .../scenarios/interuss/mock_uss/__init__.py | 0 .../interuss/mock_uss/configure_locality.md | 35 +++++ .../interuss/mock_uss/configure_locality.py | 93 +++++++++++ .../interuss/mock_uss/unconfigure_locality.md | 17 ++ .../interuss/mock_uss/unconfigure_locality.py | 51 ++++++ .../uss_qualifier/scenarios/scenario.py | 11 ++ .../suites/faa/uft/message_signing.yaml | 2 +- monitoring/uss_qualifier/suites/suite.py | 8 +- .../reports/report/ActionGeneratorReport.json | 20 ++- 36 files changed, 656 insertions(+), 49 deletions(-) create mode 100644 monitoring/mock_uss/dynamic_configuration/__init__.py create mode 100644 monitoring/mock_uss/dynamic_configuration/configuration.py create mode 100644 monitoring/mock_uss/dynamic_configuration/routes.py create mode 100644 monitoring/monitorlib/clients/mock_uss/__init__.py create mode 100644 monitoring/monitorlib/clients/mock_uss/locality.py create mode 100644 monitoring/uss_qualifier/action_generators/interuss/__init__.py create mode 100644 monitoring/uss_qualifier/action_generators/interuss/mock_uss/__init__.py create mode 100644 monitoring/uss_qualifier/action_generators/interuss/mock_uss/with_locality.py create mode 100644 monitoring/uss_qualifier/requirements/interuss/mock_uss/hosted_instance.md create mode 100644 monitoring/uss_qualifier/resources/interuss/mock_uss/__init__.py rename monitoring/uss_qualifier/resources/interuss/{mock_uss.py => mock_uss/client.py} (55%) create mode 100644 monitoring/uss_qualifier/resources/interuss/mock_uss/locality.py create mode 100644 monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py create mode 100644 monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md create mode 100644 monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py create mode 100644 monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md create mode 100644 monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py diff --git a/monitoring/mock_uss/auth.py b/monitoring/mock_uss/auth.py index 704bc0b9e0..4d4b81db39 100644 --- a/monitoring/mock_uss/auth.py +++ b/monitoring/mock_uss/auth.py @@ -7,3 +7,5 @@ webapp.config.get(config.KEY_TOKEN_PUBLIC_KEY), webapp.config.get(config.KEY_TOKEN_AUDIENCE), ) + +MOCK_USS_CONFIG_SCOPE = "interuss.mock_uss.configure" diff --git a/monitoring/mock_uss/config.py b/monitoring/mock_uss/config.py index d2371399e0..dbfd39e088 100644 --- a/monitoring/mock_uss/config.py +++ b/monitoring/mock_uss/config.py @@ -1,6 +1,5 @@ from monitoring.mock_uss import import_environment_variable from monitoring.monitorlib import auth_validation -from monitoring.monitorlib.locality import Locality KEY_TOKEN_PUBLIC_KEY = "MOCK_USS_PUBLIC_KEY" @@ -27,7 +26,5 @@ mutator=lambda s: set(svc.strip().lower() for svc in s.split(",")), ) import_environment_variable(KEY_DSS_URL, required=False) -import_environment_variable( - KEY_BEHAVIOR_LOCALITY, default="CHE", mutator=Locality.from_locale -) +import_environment_variable(KEY_BEHAVIOR_LOCALITY, default="US.IndustryCollaboration") import_environment_variable(KEY_CODE_VERSION, default="Unknown") diff --git a/monitoring/mock_uss/dynamic_configuration/__init__.py b/monitoring/mock_uss/dynamic_configuration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/mock_uss/dynamic_configuration/configuration.py b/monitoring/mock_uss/dynamic_configuration/configuration.py new file mode 100644 index 0000000000..2b883261cc --- /dev/null +++ b/monitoring/mock_uss/dynamic_configuration/configuration.py @@ -0,0 +1,29 @@ +import json + +from implicitdict import ImplicitDict +from monitoring.mock_uss import require_config_value, webapp +from monitoring.mock_uss.config import KEY_BEHAVIOR_LOCALITY +from monitoring.monitorlib.locality import Locality, LocalityCode +from monitoring.monitorlib.multiprocessing import SynchronizedValue + + +require_config_value(KEY_BEHAVIOR_LOCALITY) + + +class DynamicConfiguration(ImplicitDict): + locale: LocalityCode + + +db = SynchronizedValue( + DynamicConfiguration(locale=LocalityCode(webapp.config[KEY_BEHAVIOR_LOCALITY])), + decoder=lambda b: ImplicitDict.parse( + json.loads(b.decode("utf-8")), DynamicConfiguration + ), + capacity_bytes=10000, +) + + +def get_locality() -> Locality: + with db as tx: + code = tx.locale + return Locality.from_locale(code) diff --git a/monitoring/mock_uss/dynamic_configuration/routes.py b/monitoring/mock_uss/dynamic_configuration/routes.py new file mode 100644 index 0000000000..a6cfc66a31 --- /dev/null +++ b/monitoring/mock_uss/dynamic_configuration/routes.py @@ -0,0 +1,48 @@ +from typing import Tuple + +import flask +from implicitdict import ImplicitDict + +from monitoring.mock_uss import webapp +from monitoring.mock_uss.auth import requires_scope, MOCK_USS_CONFIG_SCOPE +from monitoring.mock_uss.dynamic_configuration.configuration import db, get_locality +from monitoring.monitorlib.clients.mock_uss.locality import ( + PutLocalityRequest, + GetLocalityResponse, +) +from monitoring.monitorlib.locality import Locality + + +@webapp.route("/configuration/locality", methods=["GET"]) +def locality_get() -> Tuple[str, int]: + return flask.jsonify( + GetLocalityResponse(locality_code=get_locality().locality_code()) + ) + + +@webapp.route("/configuration/locality", methods=["PUT"]) +@requires_scope([MOCK_USS_CONFIG_SCOPE]) # TODO: use separate public key for this +def locality_set() -> Tuple[str, int]: + """Set the locality of the mock_uss.""" + try: + json = flask.request.json + if json is None: + raise ValueError("Request did not contain a JSON payload") + req: PutLocalityRequest = ImplicitDict.parse(json, PutLocalityRequest) + except ValueError as e: + msg = f"Change locality unable to parse JSON: {str(e)}" + return msg, 400 + + # Make sure this is a valid locality + try: + Locality.from_locale(req.locality_code) + except ValueError as e: + msg = f"Invalid locality_code: {str(e)}" + return msg, 400 + + with db as tx: + tx.locale = req.locality_code + + return flask.jsonify( + GetLocalityResponse(locality_code=get_locality().locality_code()) + ) diff --git a/monitoring/mock_uss/interaction_logging/logger.py b/monitoring/mock_uss/interaction_logging/logger.py index e09acf5b3f..106b4a2399 100644 --- a/monitoring/mock_uss/interaction_logging/logger.py +++ b/monitoring/mock_uss/interaction_logging/logger.py @@ -66,7 +66,7 @@ def interaction_log_after_request(response): datetime.datetime.utcnow() - flask.current_app.custom_profiler["start"] ).total_seconds() # TODO: Make this configurable instead of hardcoding exactly these query types - if "/uss/v1/" in flask.request.url_rule.rule: + if flask.request.url_rule is not None and "/uss/v1/" in flask.request.url_rule.rule: query = describe_flask_query(flask.request, response, elapsed_s) log_interaction(QueryDirection.Incoming, query) return response diff --git a/monitoring/mock_uss/routes.py b/monitoring/mock_uss/routes.py index f423b751d3..abac76a7e0 100644 --- a/monitoring/mock_uss/routes.py +++ b/monitoring/mock_uss/routes.py @@ -49,3 +49,6 @@ def handle_exception(e): flask.jsonify({"message": "Unhandled {}: {}".format(type(e).__name__, str(e))}), 500, ) + + +from .dynamic_configuration import routes diff --git a/monitoring/mock_uss/scdsc/routes_injection.py b/monitoring/mock_uss/scdsc/routes_injection.py index e55f407b1e..182f8a5a41 100644 --- a/monitoring/mock_uss/scdsc/routes_injection.py +++ b/monitoring/mock_uss/scdsc/routes_injection.py @@ -2,7 +2,6 @@ import traceback from datetime import datetime, timedelta import time -from functools import wraps from typing import List, Tuple import uuid @@ -10,7 +9,8 @@ from implicitdict import ImplicitDict, StringBasedDateTime from loguru import logger import requests.exceptions -from uas_standards.interuss.automated_testing.scd.v1 import api as scd_api + +from monitoring.mock_uss.dynamic_configuration.configuration import get_locality from uas_standards.interuss.automated_testing.scd.v1.api import ( InjectFlightRequest, InjectFlightResponse, @@ -42,7 +42,7 @@ from monitoring.mock_uss.scdsc.routes_scdsc import op_intent_from_flightrecord from monitoring.monitorlib.geo import Polygon from monitoring.monitorlib.geotemporal import Volume4D, Volume4DCollection -from monitoring.mock_uss.config import KEY_BASE_URL, KEY_BEHAVIOR_LOCALITY +from monitoring.mock_uss.config import KEY_BASE_URL from monitoring.monitorlib import versioning from monitoring.monitorlib.clients import scd as scd_client from monitoring.monitorlib.fetch import QueryError @@ -56,7 +56,6 @@ require_config_value(KEY_BASE_URL) -require_config_value(KEY_BEHAVIOR_LOCALITY) DEADLOCK_TIMEOUT = timedelta(seconds=5) @@ -181,7 +180,7 @@ def scdsc_inject_flight(flight_id: str) -> Tuple[str, int]: def inject_flight(flight_id: str, req_body: InjectFlightRequest) -> Tuple[dict, int]: pid = os.getpid() - locality = webapp.config[KEY_BEHAVIOR_LOCALITY] + locality = get_locality() def log(msg: str): logger.debug(f"[inject_flight/{pid}:{flight_id}] {msg}") diff --git a/monitoring/monitorlib/clients/mock_uss/__init__.py b/monitoring/monitorlib/clients/mock_uss/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/monitorlib/clients/mock_uss/locality.py b/monitoring/monitorlib/clients/mock_uss/locality.py new file mode 100644 index 0000000000..839f5c034a --- /dev/null +++ b/monitoring/monitorlib/clients/mock_uss/locality.py @@ -0,0 +1,15 @@ +from implicitdict import ImplicitDict + +from monitoring.monitorlib.locality import LocalityCode + + +class PutLocalityRequest(ImplicitDict): + """API object to request a change in locality""" + + locality_code: LocalityCode + + +class GetLocalityResponse(ImplicitDict): + """API object defining a response indicating locality""" + + locality_code: LocalityCode diff --git a/monitoring/monitorlib/infrastructure.py b/monitoring/monitorlib/infrastructure.py index 5c8f1de086..093ef73c28 100644 --- a/monitoring/monitorlib/infrastructure.py +++ b/monitoring/monitorlib/infrastructure.py @@ -115,11 +115,8 @@ def adjust_request_kwargs(self, kwargs): def auth( prepared_request: requests.PreparedRequest, ) -> requests.PreparedRequest: - if not scopes: - raise ValueError( - "All tests must specify auth scope for all session requests. Either specify as an argument for each individual HTTP call, or decorate the test with @default_scope." - ) - self.auth_adapter.add_headers(prepared_request, scopes) + if scopes: + self.auth_adapter.add_headers(prepared_request, scopes) return prepared_request kwargs["auth"] = auth diff --git a/monitoring/monitorlib/locality.py b/monitoring/monitorlib/locality.py index 6e90ecd055..5fcdba720e 100644 --- a/monitoring/monitorlib/locality.py +++ b/monitoring/monitorlib/locality.py @@ -11,6 +11,12 @@ class Locality(ABC): _NOT_IMPLEMENTED_MSG = "All methods of base Locality class must be implemented by each specific subclass" + @classmethod + def locality_code(cls) -> str: + raise NotImplementedError( + "locality_code classmethod must be overridden by each specific subclass" + ) + @abstractmethod def is_uspace_applicable(self) -> bool: """Returns true iff U-space rules apply to this locality""" @@ -18,7 +24,7 @@ def is_uspace_applicable(self) -> bool: @abstractmethod def allows_same_priority_intersections(self, priority: int) -> bool: - """Returns true iff locality allows intersections between two operations at this priority level""" + """Returns true iff locality allows intersections between two operations at this priority level for ASTM F3548-21""" raise NotImplementedError(Locality._NOT_IMPLEMENTED_MSG) def __str__(self): @@ -28,25 +34,36 @@ def __str__(self): def from_locale(locality_code: LocalityCode) -> LocalityType: current_module = sys.modules[__name__] for name, obj in inspect.getmembers(current_module, inspect.isclass): - if name == locality_code: - if not issubclass(obj, Locality): - raise ValueError( - f"Locality '{name}' is not a subclass of the Locality abstract base class" - ) - return obj() + if issubclass(obj, Locality) and obj != Locality: + if obj.locality_code() == locality_code: + return obj() raise ValueError( - f"Could not find Locality implementation for Locality code '{locality_code}' (expected to find a subclass of the Locality astract base class named {locality_code})" + f"Could not find Locality implementation for Locality code '{locality_code}' (expected to find a subclass of the Locality astract base class where classmethod locality_code returns '{locality_code}')" ) LocalityType = TypeVar("LocalityType", bound=Locality) -class CHE(Locality): - """Switzerland""" +class Switzerland(Locality): + @classmethod + def locality_code(cls) -> str: + return "CHE" def is_uspace_applicable(self) -> bool: return True def allows_same_priority_intersections(self, priority: int) -> bool: return False + + +class UnitedStatesIndustryCollaboration(Locality): + @classmethod + def locality_code(cls) -> str: + return "US.IndustryCollaboration" + + def is_uspace_applicable(self) -> bool: + return False + + def allows_same_priority_intersections(self, priority: int) -> bool: + return False diff --git a/monitoring/uss_qualifier/action_generators/interuss/__init__.py b/monitoring/uss_qualifier/action_generators/interuss/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/action_generators/interuss/mock_uss/__init__.py b/monitoring/uss_qualifier/action_generators/interuss/mock_uss/__init__.py new file mode 100644 index 0000000000..a411c78f33 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/interuss/mock_uss/__init__.py @@ -0,0 +1 @@ +from .with_locality import WithLocality diff --git a/monitoring/uss_qualifier/action_generators/interuss/mock_uss/with_locality.py b/monitoring/uss_qualifier/action_generators/interuss/mock_uss/with_locality.py new file mode 100644 index 0000000000..96ed7a79d5 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/interuss/mock_uss/with_locality.py @@ -0,0 +1,148 @@ +from typing import Dict, List, Optional + +from implicitdict import ImplicitDict +from monitoring.monitorlib.inspection import fullname +from monitoring.uss_qualifier.action_generators.documentation.definitions import ( + PotentialGeneratedAction, + PotentialTestScenarioAction, +) +from monitoring.uss_qualifier.action_generators.documentation.documentation import ( + list_potential_actions_for_action_declaration, +) +from monitoring.uss_qualifier.reports.report import TestSuiteActionReport +from monitoring.uss_qualifier.resources.definitions import ResourceID +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSsResource +from monitoring.uss_qualifier.resources.interuss.mock_uss.locality import ( + LocalityResource, +) +from monitoring.uss_qualifier.resources.resource import ResourceType +from monitoring.uss_qualifier.scenarios.definitions import TestScenarioDeclaration +from monitoring.uss_qualifier.scenarios.interuss.mock_uss.configure_locality import ( + ConfigureLocality, +) +from monitoring.uss_qualifier.scenarios.interuss.mock_uss.unconfigure_locality import ( + UnconfigureLocality, +) +from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_name + +from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration +from monitoring.uss_qualifier.suites.suite import ( + ActionGenerator, + TestSuiteAction, + ReactionToFailure, +) + + +class WithLocalitySpecification(ImplicitDict): + action_to_wrap: TestSuiteActionDeclaration + """Test suite action to perform after setting mock_uss localities""" + + mock_uss_instances_source: ResourceID + """ID of the resource providing all mock_uss instances to change the locality of""" + + locality_source: ResourceID + """ID of the resource providing the locality to use temporarily for the provided mock_uss instances""" + + +class WithLocality(ActionGenerator[WithLocalitySpecification]): + """Performs a specified test suite action after first configuring mock_uss instances to use a specified locality, + and then restoring the original locality afterward.""" + + _actions: List[TestSuiteAction] + _current_action: int + _failure_reaction: ReactionToFailure + + @classmethod + def list_potential_actions( + cls, specification: WithLocalitySpecification + ) -> List[PotentialGeneratedAction]: + actions = [ + PotentialGeneratedAction( + test_scenario=PotentialTestScenarioAction( + scenario_type=get_scenario_type_name(ConfigureLocality) + ) + ) + ] + actions.extend( + list_potential_actions_for_action_declaration(specification.action_to_wrap) + ) + actions.append( + PotentialGeneratedAction( + test_scenario=PotentialTestScenarioAction( + scenario_type=get_scenario_type_name(UnconfigureLocality) + ) + ) + ) + return actions + + def __init__( + self, + specification: WithLocalitySpecification, + resources: Dict[ResourceID, ResourceType], + ): + if specification.mock_uss_instances_source not in resources: + raise ValueError( + f"Missing mock_uss_instances_source resource ID '{specification.mock_uss_instances_source}' in resource pool" + ) + if not isinstance( + resources[specification.mock_uss_instances_source], MockUSSsResource + ): + raise ValueError( + f"mock_uss_instances_source resource '{specification.mock_uss_instances_source}' is a {fullname(type(resources[specification.mock_uss_instances_source]))} rather than the expected {fullname(MockUSSsResource)}" + ) + if specification.locality_source not in resources: + raise ValueError( + f"Missing locality_source resource ID '{specification.locality_source}' in resource pool" + ) + if not isinstance(resources[specification.locality_source], LocalityResource): + raise ValueError( + f"locality_source resource '{specification.locality_source}' is a {fullname(type(resources[specification.locality_source]))} rather than the expected {fullname(LocalityResource)}" + ) + + # Continue to unconfigure localities even for failure in the main action + action_to_wrap = ImplicitDict.parse( + specification.action_to_wrap, TestSuiteActionDeclaration + ) + action_to_wrap.on_failure = ReactionToFailure.Continue + + self._actions = [ + TestSuiteAction( + TestSuiteActionDeclaration( + test_scenario=TestScenarioDeclaration( + scenario_type=get_scenario_type_name(ConfigureLocality), + resources={ + "mock_uss_instances": specification.mock_uss_instances_source, + "locality": specification.locality_source, + }, + ), + on_failure=ReactionToFailure.Abort, + ), + resources, + ), + TestSuiteAction(action_to_wrap, resources), + TestSuiteAction( + TestSuiteActionDeclaration( + test_scenario=TestScenarioDeclaration( + scenario_type=get_scenario_type_name(UnconfigureLocality), + resources={}, + ), + on_failure=ReactionToFailure.Continue, + ), + resources, + ), + ] + self._current_action = 0 + + def run_next_action(self) -> Optional[TestSuiteActionReport]: + from loguru import logger + + logger.debug(f"run_next_action with current action {self._current_action}") + if self._current_action < len(self._actions): + report = self._actions[self._current_action].run() + if not report.successful() and self._current_action == 0: + self._current_action = len(self._actions) + else: + self._current_action += 1 + return report + else: + return None diff --git a/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml b/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml index 8800f03254..84510a893a 100644 --- a/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/faa/uft/local_message_signing.yaml @@ -40,7 +40,7 @@ v1: participant_id: uss1 base_url: http://host.docker.internal:8082 mock_uss: - resource_type: resources.interuss.MockUSSResource + resource_type: resources.interuss.mock_uss.client.MockUSSResource dependencies: auth_adapter: utm_auth specification: diff --git a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml index ab16eec8aa..3aaf1f9b14 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml @@ -169,3 +169,16 @@ f3548_single_scenario: participant_id: uss2 injection_base_url: http://scdsc.uss2.localutm/scdsc local_debug: true + +mock_uss_instances: + mock_uss_instances_scdsc: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.interuss.mock_uss.client.MockUSSsResource + dependencies: + auth_adapter: utm_auth + specification: + instances: + - mock_uss_base_url: http://scdsc.uss1.localutm + participant_id: uss1 + - mock_uss_base_url: http://scdsc.uss2.localutm + participant_id: uss2 diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index 324a0b5d61..3fd555c5aa 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -4,6 +4,7 @@ all: - $ref: '#/net_rid_sims' - $ref: '#/general_flight_authorization' - $ref: '#/geospatial_map' + - $ref: '#/mock_uss_instances' uspace: allOf: @@ -339,3 +340,11 @@ geospatial_map: expected_result: Advise common: $ref: 'environment.yaml#/common' + +mock_uss_instances: + $ref: 'environment.yaml#/mock_uss_instances' + locality_che: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.interuss.mock_uss.locality.LocalityResource + specification: + locality_code: CHE diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index e7b54c7711..1030f5d36f 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -6,9 +6,12 @@ v1: resource_declarations: $ref: ./library/resources.yaml#/all action: - test_suite: - suite_type: suites.uspace.required_services + action_generator: + generator_type: action_generators.interuss.mock_uss.WithLocality resources: + mock_uss_instances: mock_uss_instances_scdsc + locality: locality_che + conflicting_flights: conflicting_flights priority_preemption_flights: priority_preemption_flights invalid_flight_intents: invalid_flight_intents @@ -23,6 +26,28 @@ v1: netrid_dss_instances: netrid_dss_instances_v22a id_generator: id_generator service_area: service_area + specification: + mock_uss_instances_source: mock_uss_instances + locality_source: locality + action_to_wrap: + test_suite: + suite_type: suites.uspace.required_services + resources: + conflicting_flights: conflicting_flights + priority_preemption_flights: priority_preemption_flights + invalid_flight_intents: invalid_flight_intents + invalid_flight_auth_flights: invalid_flight_auth_flights + flight_planners: flight_planners + dss: dss + + flights_data: flights_data + service_providers: service_providers + observers: observers + evaluation_configuration: evaluation_configuration + netrid_dss_instances: netrid_dss_instances + id_generator: id_generator + service_area: service_area + on_failure: Continue artifacts: tested_roles: report_path: output/tested_roles_uspace diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index b0e1a2bef4..671a78552d 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -304,12 +304,17 @@ class ActionGeneratorReport(ImplicitDict): generator_type: GeneratorTypeName """Type of action generator""" + start_time: StringBasedDateTime + """Time at which the action generator started""" + actions: List[TestSuiteActionReport] """Reports from the actions generated by the action generator, in order of execution.""" - @property - def successful(self) -> bool: - return all(a.successful() for a in self.actions) + end_time: Optional[StringBasedDateTime] + """Time at which the action generator completed or encountered an error""" + + successful: bool = False + """True iff all actions completed normally with no failed checks""" def has_critical_problem(self) -> bool: return any(a.has_critical_problem() for a in self.actions) @@ -346,18 +351,6 @@ def participant_ids(self) -> Set[ParticipantID]: ids.update(action.participant_ids()) return ids - @property - def start_time(self) -> Optional[StringBasedDateTime]: - if not self.actions: - return None - return self.actions[0].start_time - - @property - def end_time(self) -> Optional[StringBasedDateTime]: - if not self.actions: - return None - return self.actions[-1].end_time - class TestSuiteActionReport(ImplicitDict): test_suite: Optional[TestSuiteReport] diff --git a/monitoring/uss_qualifier/requirements/interuss/mock_uss/hosted_instance.md b/monitoring/uss_qualifier/requirements/interuss/mock_uss/hosted_instance.md new file mode 100644 index 0000000000..d96e566e1f --- /dev/null +++ b/monitoring/uss_qualifier/requirements/interuss/mock_uss/hosted_instance.md @@ -0,0 +1,11 @@ +# InterUSS mock_uss instance requirements + +## Overview + +These requirements apply to a test participant providing a hosted instance of InterUSS's mock_uss. + +## Requirements + +### ExposeInterface + +A test participant providing a hosted instance of InterUSS's mock_uss must expose all endpoints implemented by mock_uss, as configured by the test participant, appropriate to the tests being performed. All mock_uss endpoints necessary for the tests being performed must be exposed and must behave according to the InterUSS codebase version appropriate to the tests. diff --git a/monitoring/uss_qualifier/resources/interuss/__init__.py b/monitoring/uss_qualifier/resources/interuss/__init__.py index 6867df15f4..388371305d 100644 --- a/monitoring/uss_qualifier/resources/interuss/__init__.py +++ b/monitoring/uss_qualifier/resources/interuss/__init__.py @@ -1,2 +1 @@ from .id_generator import IDGeneratorResource -from .mock_uss import MockUSSResource diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss/__init__.py b/monitoring/uss_qualifier/resources/interuss/mock_uss/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss.py b/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py similarity index 55% rename from monitoring/uss_qualifier/resources/interuss/mock_uss.py rename to monitoring/uss_qualifier/resources/interuss/mock_uss/client.py index e46c13ad18..e70ab45d10 100644 --- a/monitoring/uss_qualifier/resources/interuss/mock_uss.py +++ b/monitoring/uss_qualifier/resources/interuss/mock_uss/client.py @@ -1,6 +1,14 @@ +from typing import Tuple, Optional, List + from implicitdict import ImplicitDict + from monitoring.monitorlib import fetch +from monitoring.monitorlib.clients.mock_uss.locality import ( + GetLocalityResponse, + PutLocalityRequest, +) from monitoring.monitorlib.infrastructure import AuthAdapter, UTMClientSession +from monitoring.monitorlib.locality import LocalityCode from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( SCOPE_SCD_QUALIFIER_INJECT, ) @@ -9,6 +17,9 @@ from monitoring.uss_qualifier.resources.resource import Resource +MOCK_USS_CONFIG_SCOPE = "interuss.mock_uss.configure" + + class MockUSSClient(object): """Means to communicate with an InterUSS mock_uss instance""" @@ -30,6 +41,31 @@ def get_status(self) -> fetch.Query: server_id=self.participant_id, ) + def get_locality(self) -> Tuple[Optional[LocalityCode], fetch.Query]: + query = fetch.query_and_describe( + self.session, + "GET", + "/configuration/locality", + server_id=self.participant_id, + ) + if query.status_code != 200: + return None, query + try: + resp = ImplicitDict.parse(query.response.json, GetLocalityResponse) + except ValueError: + return None, query + return resp.locality_code, query + + def set_locality(self, locality_code: LocalityCode) -> fetch.Query: + return fetch.query_and_describe( + self.session, + "PUT", + "/configuration/locality", + scope=MOCK_USS_CONFIG_SCOPE, + server_id=self.participant_id, + json=PutLocalityRequest(locality_code=locality_code), + ) + # TODO: Add other methods to interact with the mock USS in other ways (like starting/stopping message signing data collection) @@ -60,3 +96,19 @@ def __init__( specification.mock_uss_base_url, auth_adapter.adapter, ) + + +class MockUSSsSpecification(ImplicitDict): + instances: List[MockUSSSpecification] + + +class MockUSSsResource(Resource[MockUSSsSpecification]): + mock_uss_instances: List[MockUSSClient] + + def __init__( + self, specification: MockUSSsSpecification, auth_adapter: AuthAdapterResource + ): + self.mock_uss_instances = [ + MockUSSClient(s.participant_id, s.mock_uss_base_url, auth_adapter.adapter) + for s in specification.instances + ] diff --git a/monitoring/uss_qualifier/resources/interuss/mock_uss/locality.py b/monitoring/uss_qualifier/resources/interuss/mock_uss/locality.py new file mode 100644 index 0000000000..0871db919a --- /dev/null +++ b/monitoring/uss_qualifier/resources/interuss/mock_uss/locality.py @@ -0,0 +1,18 @@ +from implicitdict import ImplicitDict + +from monitoring.monitorlib.locality import LocalityCode, Locality +from monitoring.uss_qualifier.resources.resource import Resource + + +class LocalitySpecification(ImplicitDict): + locality_code: LocalityCode + + +class LocalityResource(Resource[LocalitySpecification]): + locality_code: LocalityCode + + def __init__(self, specification: LocalitySpecification): + # Make sure provided code is valid + Locality.from_locale(specification.locality_code) + + self.locality_code = specification.locality_code diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py index 2f837bee6e..d819ce33fa 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py @@ -1,5 +1,5 @@ from monitoring.uss_qualifier.common_data_definitions import Severity -from monitoring.uss_qualifier.resources.interuss.mock_uss import ( +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( MockUSSResource, MockUSSClient, ) diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py index 19229a5da2..2180aadd28 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py @@ -1,5 +1,5 @@ from monitoring.uss_qualifier.common_data_definitions import Severity -from monitoring.uss_qualifier.resources.interuss.mock_uss import ( +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( MockUSSResource, MockUSSClient, ) diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md new file mode 100644 index 0000000000..cf2a4ec27e --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md @@ -0,0 +1,35 @@ +# Configure mock_uss locality test scenario + +This test scenario instructs a collection of mock USS instances to use a specified locality. The old locality is recorded so that the [UnconfigureLocality test scenario](./unconfigure_locality.md) can set the locality values back to what they were before. + +This scenario should not generally be used directly; instead, the [WithLocality action generator](../../../action_generators/interuss/mock_uss/with_locality.py) should be used to temporarily change the locality of mock_uss instances when appropriate. + +## Resources + +### mock_uss_instances + +The means to communicate with the mock USS instances that will have their localities set. + +### locality + +The locality to set all mock USS instances to. + +## Set locality test case + +### Get current locality value test step + +#### Query ok check + +If a mock USS instance doesn't respond properly to a request to get its current locality, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. + +### Set locality to desired value test step + +#### Query ok check + +If a mock USS instance doesn't respond properly to a request to change its locality, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. + +## Cleanup + +### Restore locality check + +If uss_qualifier cannot restore a mock_uss instance's locality to its old value when rolling back incomplete locality changes, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py new file mode 100644 index 0000000000..fea51bb90c --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py @@ -0,0 +1,93 @@ +from typing import List + +from monitoring.monitorlib.locality import LocalityCode +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( + MockUSSsResource, + MockUSSClient, +) +from monitoring.uss_qualifier.resources.interuss.mock_uss.locality import ( + LocalityResource, +) +from monitoring.uss_qualifier.scenarios.interuss.mock_uss.unconfigure_locality import ( + MockUSSLocalityConfiguration, + unconfigure_stack, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class ConfigureLocality(TestScenario): + mock_uss_instances: List[MockUSSClient] + locality_code: LocalityCode + to_unconfigure: List[MockUSSLocalityConfiguration] + + def __init__( + self, mock_uss_instances: MockUSSsResource, locality: LocalityResource + ): + super(ConfigureLocality, self).__init__() + self.mock_uss_instances = mock_uss_instances.mock_uss_instances + self.locality_code = locality.locality_code + self.to_unconfigure = [] + + def run(self): + self.begin_test_scenario() + + self.begin_test_case("Set locality") + + self.begin_test_step("Get current locality value") + old_locality_codes = {} + for mock_uss in self.mock_uss_instances: + locality_code, query = mock_uss.get_locality() + self.record_query(query) + with self.check("Query ok", [mock_uss.participant_id]) as check: + if query.status_code != 200: + check.record_failed( + f"Get current locality returned {query.status_code}", + Severity.High, + query_timestamps=[query.request.initiated_at.datetime], + ) + elif locality_code is None: + check.record_failed( + f"Missing current locality code", + Severity.High, + "Query to get current locality value did not produce a valid locality code", + query_timestamps=[query.request.initiated_at.datetime], + ) + old_locality_codes[mock_uss] = locality_code + self.record_note( + mock_uss.session.get_prefix_url() + " old locality", locality_code + ) + self.end_test_step() + + self.begin_test_step("Set locality to desired value") + for mock_uss in self.mock_uss_instances: + query = mock_uss.set_locality(self.locality_code) + self.record_query(query) + with self.check("Query ok", [mock_uss.participant_id]) as check: + if query.status_code != 200: + check.record_failed( + f"Set locality returned {query.status_code}", + Severity.High, + query_timestamps=[query.request.initiated_at.datetime], + ) + self.to_unconfigure.append( + MockUSSLocalityConfiguration( + client=mock_uss, locality_code=old_locality_codes[mock_uss] + ) + ) + unconfigure_stack.append(self.to_unconfigure) + self.to_unconfigure = [] + self.end_test_step() + + self.end_test_case() + + self.end_test_scenario() + + def cleanup(self): + self.begin_cleanup() + + for instance in self.to_unconfigure: + query = instance.client.set_locality(instance.locality_code) + self.record_query(query) + + self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md new file mode 100644 index 0000000000..e679e7dde7 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md @@ -0,0 +1,17 @@ +# Unconfigure mock_uss locality test scenario + +This test scenario restores the locality setting for a collection of mock USS instances following a [ConfigureLocality test scenario](./configure_locality.md). + +## Resources + +No resources are needed because they are stored by the [ConfigureLocality test scenario](./configure_locality.md). + +## Restore locality test case + +### Set locality to old value test step + +The most recent ConfigureLocality test scenario instances recorded the old locality values and mock_uss instances. This test step consumes that information to restore localities to their old values. + +#### Query ok check + +If a mock USS instance doesn't respond properly to a request to change its locality, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py new file mode 100644 index 0000000000..18ffaafb28 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass +from typing import List + +from monitoring.monitorlib.locality import LocalityCode +from monitoring.uss_qualifier.common_data_definitions import Severity +from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +@dataclass +class MockUSSLocalityConfiguration(object): + client: MockUSSClient + locality_code: LocalityCode + + +unconfigure_stack: List[List[MockUSSLocalityConfiguration]] = [] +"""The stack of mock_uss locality configurations that have been performed by configure_locality. + +UnconfigureLocality will reset localities according to the most recent stack addition.""" + + +class UnconfigureLocality(TestScenario): + def run(self): + self.begin_test_scenario() + + if not unconfigure_stack: + raise ValueError( + "UnconfigureLocality attempted to access an empty stack of locality configurations; ConfigureLocality must be run first and UnconfigureLocality instances may not exceed ConfigureLocality instances" + ) + to_unconfigure = unconfigure_stack.pop(-1) + + self.begin_test_case("Restore locality") + + self.begin_test_step("Set locality to old value") + + for instance in to_unconfigure: + query = instance.client.set_locality(instance.locality_code) + self.record_query(query) + with self.check("Query ok", [instance.client.participant_id]) as check: + if query.status_code != 200: + check.record_failed( + f"Set locality returned {query.status_code}", + Severity.Medium, + query_timestamps=[query.request.initiated_at.datetime], + ) + + self.end_test_step() + + self.end_test_case() + + self.end_test_scenario() diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 2ee5aec44b..5cd0f44b87 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -509,6 +509,17 @@ class TestScenario(GenericTestScenario): pass +def get_scenario_type_name(scenario_type: Type[TestScenario]) -> TestScenarioTypeName: + full_name = fullname(scenario_type) + if not issubclass(scenario_type, TestScenario): + raise ValueError(f"{full_name} is not a TestScenario") + if not full_name.startswith("monitoring.uss_qualifier.scenarios"): + raise ValueError( + f"{full_name} does not appear to be located in the standard root path for test scenarios" + ) + return TestScenarioTypeName(full_name[len("monitoring.uss_qualifier.") :]) + + TestScenarioType = TypeVar("TestScenarioType", bound=TestScenario) diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml b/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml index 33c39c8611..e55b112741 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml @@ -1,6 +1,6 @@ name: UFT message signing resources: - mock_uss: resources.interuss.MockUSSResource + mock_uss: resources.interuss.mock_uss.client.MockUSSResource flight_planners: resources.flight_planning.FlightPlannersResource combination_selector: resources.flight_planning.FlightPlannerCombinationSelectorResource dss: resources.astm.f3548.v21.DSSInstanceResource diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index a9fe19d201..00ab9d2f1b 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -5,6 +5,8 @@ import json from typing import Dict, List, Optional +import arrow + from implicitdict import StringBasedDateTime, ImplicitDict from loguru import logger import yaml @@ -139,7 +141,9 @@ def _run_test_suite(self) -> TestSuiteReport: def _run_action_generator(self) -> ActionGeneratorReport: report = ActionGeneratorReport( - actions=[], generator_type=self.action_generator.definition.generator_type + actions=[], + generator_type=self.action_generator.definition.generator_type, + start_time=StringBasedDateTime(arrow.utcnow()), ) while True: action_report = self.action_generator.run_next_action() @@ -148,6 +152,8 @@ def _run_action_generator(self) -> ActionGeneratorReport: report.actions.append(action_report) if action_report.has_critical_problem(): break + report.end_time = StringBasedDateTime(arrow.utcnow()) + report.successful = all(a.successful() for a in report.actions) return report diff --git a/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json b/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json index 135bc8f2ee..83de506305 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json @@ -14,14 +14,32 @@ }, "type": "array" }, + "end_time": { + "description": "Time at which the action generator completed or encountered an error", + "format": "date-time", + "type": [ + "string", + "null" + ] + }, "generator_type": { "description": "Type of action generator", "type": "string" + }, + "start_time": { + "description": "Time at which the action generator started", + "format": "date-time", + "type": "string" + }, + "successful": { + "description": "True iff all actions completed normally with no failed checks", + "type": "boolean" } }, "required": [ "actions", - "generator_type" + "generator_type", + "start_time" ], "type": "object" } \ No newline at end of file From 5d7821e777075b1d14693053022c899c5f90d6bc Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Wed, 11 Oct 2023 15:12:43 -0700 Subject: [PATCH 3/3] [monitorlib] Add generic flight_planning client (#234) * Add flight planning client * Use flight planning client * Update flight intent validation * Fix behavior * Re-fix off-nominal op intent treatment --- monitoring/mock_uss/scdsc/flight_planning.py | 9 + monitoring/mock_uss/scdsc/routes_injection.py | 6 +- .../clients/flight_planning/__init__.py | 0 .../clients/flight_planning/client.py | 94 ++++++ .../clients/flight_planning/client_scd.py | 272 ++++++++++++++++++ .../clients/flight_planning/flight_info.py | 236 +++++++++++++++ .../flight_planning/flight_info_template.py | 53 ++++ .../clients/flight_planning/planning.py | 71 +++++ .../flight_planning/test_preparation.py | 25 ++ monitoring/monitorlib/geotemporal.py | 23 ++ .../flight_planning/flight_planner.py | 237 ++++++++------- .../flight_intent_validation.md | 65 ----- .../flight_intent_validation.py | 140 +-------- .../conflict_equal_priority_not_permitted.py | 10 +- .../utm/validate_shared_operational_intent.md | 2 +- .../prioritization_test_steps.py | 40 ++- .../scenarios/flight_planning/test_steps.py | 8 +- 17 files changed, 974 insertions(+), 317 deletions(-) create mode 100644 monitoring/monitorlib/clients/flight_planning/__init__.py create mode 100644 monitoring/monitorlib/clients/flight_planning/client.py create mode 100644 monitoring/monitorlib/clients/flight_planning/client_scd.py create mode 100644 monitoring/monitorlib/clients/flight_planning/flight_info.py create mode 100644 monitoring/monitorlib/clients/flight_planning/flight_info_template.py create mode 100644 monitoring/monitorlib/clients/flight_planning/planning.py create mode 100644 monitoring/monitorlib/clients/flight_planning/test_preparation.py diff --git a/monitoring/mock_uss/scdsc/flight_planning.py b/monitoring/mock_uss/scdsc/flight_planning.py index 09d63cd76f..b63b47d9b9 100644 --- a/monitoring/mock_uss/scdsc/flight_planning.py +++ b/monitoring/mock_uss/scdsc/flight_planning.py @@ -9,6 +9,7 @@ from monitoring.monitorlib.geotemporal import Volume4DCollection from monitoring.monitorlib.locality import Locality from monitoring.monitorlib.uspace import problems_with_flight_authorisation +from uas_standards.interuss.automated_testing.scd.v1.api import OperationalIntentState class PlanningError(Exception): @@ -47,6 +48,7 @@ def validate_request(req_body: scd_api.InjectFlightRequest, locality: Locality) # Validate max planning horizon for creation start_time = Volume4DCollection.from_interuss_scd_api( req_body.operational_intent.volumes + + req_body.operational_intent.off_nominal_volumes ).time_start.datetime time_delta = start_time - datetime.now(tz=start_time.tzinfo) if ( @@ -87,6 +89,13 @@ def check_for_disallowed_conflicts( if log is None: log = lambda msg: None + if req_body.operational_intent.state not in ( + OperationalIntentState.Accepted, + OperationalIntentState.Activated, + ): + # No conflicts are disallowed if the flight is not nominal + return + v1 = Volume4DCollection.from_interuss_scd_api(req_body.operational_intent.volumes) for op_intent in op_intents: diff --git a/monitoring/mock_uss/scdsc/routes_injection.py b/monitoring/mock_uss/scdsc/routes_injection.py index 182f8a5a41..138fbe9c82 100644 --- a/monitoring/mock_uss/scdsc/routes_injection.py +++ b/monitoring/mock_uss/scdsc/routes_injection.py @@ -244,9 +244,11 @@ def log(msg: str): # Check for operational intents in the DSS step_name = "querying for operational intents" log("Obtaining latest operational intent information") - vol4 = Volume4DCollection.from_interuss_scd_api( + v1 = Volume4DCollection.from_interuss_scd_api( req_body.operational_intent.volumes - ).bounding_volume.to_f3548v21() + + req_body.operational_intent.off_nominal_volumes + ) + vol4 = v1.bounding_volume.to_f3548v21() op_intents = query_operational_intents(vol4) # Check for intersections diff --git a/monitoring/monitorlib/clients/flight_planning/__init__.py b/monitoring/monitorlib/clients/flight_planning/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/monitorlib/clients/flight_planning/client.py b/monitoring/monitorlib/clients/flight_planning/client.py new file mode 100644 index 0000000000..e64a242efb --- /dev/null +++ b/monitoring/monitorlib/clients/flight_planning/client.py @@ -0,0 +1,94 @@ +from abc import ABC, abstractmethod +from typing import List, Optional, Union + +from monitoring.monitorlib.clients.flight_planning.test_preparation import ( + TestPreparationActivityResponse, +) + +from monitoring.monitorlib.clients.flight_planning.flight_info import ( + FlightInfo, + FlightID, + ExecutionStyle, +) +from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResponse, +) +from monitoring.monitorlib.fetch import Query +from monitoring.monitorlib.geotemporal import Volume4D + + +class PlanningActivityError(Exception): + queries: List[Query] + + def __init__( + self, message: str, queries: Optional[Union[Query, List[Query]]] = None + ): + super(PlanningActivityError, self).__init__(message) + if queries is None: + self.queries = [] + elif isinstance(queries, Query): + self.queries = [queries] + else: + self.queries = queries + + +class FlightPlannerClient(ABC): + """Client to interact with a USS as a user performing flight planning activities and as the test director preparing for tests involving flight planning activities.""" + + # ===== Emulation of user actions ===== + + @abstractmethod + def try_plan_flight( + self, flight_info: FlightInfo, execution_style: ExecutionStyle + ) -> PlanningActivityResponse: + """Instruct the USS to emulate a normal user trying to plan the described flight. + + Raises: + * PlanningActivityError + """ + raise NotImplementedError() + + @abstractmethod + def try_update_flight( + self, + flight_id: FlightID, + updated_flight_info: FlightInfo, + execution_style: ExecutionStyle, + ) -> PlanningActivityResponse: + """Instruct the USS to emulate a normal user trying to update the specified flight as described. + + Raises: + * PlanningActivityError + """ + raise NotImplementedError() + + @abstractmethod + def try_end_flight( + self, flight_id: FlightID, execution_style: ExecutionStyle + ) -> PlanningActivityResponse: + """Instruct the USS to emulate a normal user trying to end the specified flight. + + Raises: + * PlanningActivityError + """ + raise NotImplementedError() + + # ===== Test preparation activities ===== + + @abstractmethod + def report_readiness(self) -> TestPreparationActivityResponse: + """Acting as test director, ask the USS about its readiness to use its flight planning interface for automated testing. + + Raises: + * PlanningActivityError + """ + raise NotImplementedError() + + @abstractmethod + def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: + """Acting as test director, instruct the USS to close/end/remove all flights it manages within the specified area. + + Raises: + * PlanningActivityError + """ + raise NotImplementedError() diff --git a/monitoring/monitorlib/clients/flight_planning/client_scd.py b/monitoring/monitorlib/clients/flight_planning/client_scd.py new file mode 100644 index 0000000000..f5eb4643c7 --- /dev/null +++ b/monitoring/monitorlib/clients/flight_planning/client_scd.py @@ -0,0 +1,272 @@ +import uuid +from typing import Dict + +from implicitdict import ImplicitDict +from monitoring.monitorlib.clients.flight_planning.client import ( + FlightPlannerClient, + PlanningActivityError, +) +from monitoring.monitorlib.clients.flight_planning.test_preparation import ( + TestPreparationActivityResponse, +) +from uas_standards.interuss.automated_testing.scd.v1 import api as scd_api +from uas_standards.interuss.automated_testing.scd.v1 import ( + constants as scd_api_constants, +) + +from monitoring.monitorlib.clients.flight_planning.flight_info import ( + FlightInfo, + FlightID, + AirspaceUsageState, + UasState, + ExecutionStyle, +) +from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResponse, + PlanningActivityResult, + FlightPlanStatus, +) +from monitoring.monitorlib.fetch import query_and_describe +from monitoring.monitorlib.geotemporal import Volume4D +from monitoring.monitorlib.infrastructure import UTMClientSession + + +class SCDFlightPlannerClient(FlightPlannerClient): + SCD_SCOPE = scd_api_constants.Scope.Inject + _session: UTMClientSession + _plan_statuses: Dict[FlightID, FlightPlanStatus] + + def __init__(self, session: UTMClientSession): + self._session = session + self._plan_statuses = {} + + def _inject( + self, + flight_id: FlightID, + flight_info: FlightInfo, + execution_style: ExecutionStyle, + ) -> PlanningActivityResponse: + if execution_style != ExecutionStyle.IfAllowed: + raise PlanningActivityError( + f"Legacy scd automated testing API only supports {ExecutionStyle.IfAllowed} actions; '{execution_style}' is not supported" + ) + usage_state = flight_info.basic_information.usage_state + uas_state = flight_info.basic_information.uas_state + if uas_state == UasState.Nominal: + if usage_state == AirspaceUsageState.Planned: + state = scd_api.OperationalIntentState.Accepted + elif usage_state == AirspaceUsageState.InUse: + state = scd_api.OperationalIntentState.Activated + else: + raise NotImplementedError( + f"Unsupported operator AirspaceUsageState '{usage_state}' with UasState '{uas_state}'" + ) + elif usage_state == AirspaceUsageState.InUse: + if uas_state == UasState.OffNominal: + state = scd_api.OperationalIntentState.Nonconforming + elif uas_state == UasState.Contingent: + state = scd_api.OperationalIntentState.Contingent + else: + raise NotImplementedError( + f"Unsupported operator UasState '{uas_state}' with AirspaceUsageState '{usage_state}'" + ) + else: + raise NotImplementedError( + f"Unsupported combination of operator AirspaceUsageState '{usage_state}' and UasState '{uas_state}'" + ) + + if uas_state == UasState.Nominal: + volumes = [ + v.to_interuss_scd_api() for v in flight_info.basic_information.area + ] + off_nominal_volumes = [] + else: + volumes = [] + off_nominal_volumes = [ + v.to_interuss_scd_api() for v in flight_info.basic_information.area + ] + + if "astm_f3548_21" in flight_info and flight_info.astm_f3548_21: + priority = flight_info.astm_f3548_21.priority + else: + priority = 0 + + operational_intent = scd_api.OperationalIntentTestInjection( + state=state, + priority=priority, + volumes=volumes, + off_nominal_volumes=off_nominal_volumes, + ) + + kwargs = {"operational_intent": operational_intent} + if ( + "uspace_flight_authorisation" in flight_info + and flight_info.uspace_flight_authorisation + ): + kwargs["flight_authorisation"] = ImplicitDict.parse( + flight_info.uspace_flight_authorisation, scd_api.FlightAuthorisationData + ) + req = scd_api.InjectFlightRequest(**kwargs) + + op = scd_api.OPERATIONS[scd_api.OperationID.InjectFlight] + url = op.path.format(flight_id=flight_id) + query = query_and_describe( + self._session, op.verb, url, json=req, scope=self.SCD_SCOPE + ) + if query.status_code != 200 and query.status_code != 201: + raise PlanningActivityError( + f"Attempt to plan flight returned status {query.status_code} rather than 200 as expected", + query, + ) + try: + resp: scd_api.InjectFlightResponse = ImplicitDict.parse( + query.response.json, scd_api.InjectFlightResponse + ) + except ValueError as e: + raise PlanningActivityError( + f"Response to plan flight could not be parsed: {str(e)}", query + ) + + old_state = ( + self._plan_statuses[flight_id] + if flight_id in self._plan_statuses + else FlightPlanStatus.NotPlanned + ) + response = PlanningActivityResponse( + flight_id=flight_id, + queries=[query], + activity_result={ + scd_api.InjectFlightResponseResult.Planned: PlanningActivityResult.Completed, + scd_api.InjectFlightResponseResult.ReadyToFly: PlanningActivityResult.Completed, + scd_api.InjectFlightResponseResult.ConflictWithFlight: PlanningActivityResult.Rejected, + scd_api.InjectFlightResponseResult.Rejected: PlanningActivityResult.Rejected, + scd_api.InjectFlightResponseResult.Failed: PlanningActivityResult.Failed, + scd_api.InjectFlightResponseResult.NotSupported: PlanningActivityResult.NotSupported, + }[resp.result], + flight_plan_status={ + scd_api.InjectFlightResponseResult.Planned: FlightPlanStatus.Planned, + scd_api.InjectFlightResponseResult.ReadyToFly: FlightPlanStatus.OkToFly, + scd_api.InjectFlightResponseResult.ConflictWithFlight: old_state, + scd_api.InjectFlightResponseResult.Rejected: old_state, + scd_api.InjectFlightResponseResult.Failed: old_state, + scd_api.InjectFlightResponseResult.NotSupported: old_state, + }[resp.result], + ) + self._plan_statuses[flight_id] = response.flight_plan_status + return response + + def try_plan_flight( + self, flight_info: FlightInfo, execution_style: ExecutionStyle + ) -> PlanningActivityResponse: + return self._inject(str(uuid.uuid4()), flight_info, execution_style) + + def try_update_flight( + self, + flight_id: FlightID, + updated_flight_info: FlightInfo, + execution_style: ExecutionStyle, + ) -> PlanningActivityResponse: + return self._inject(flight_id, updated_flight_info, execution_style) + + def try_end_flight( + self, flight_id: FlightID, execution_style: ExecutionStyle + ) -> PlanningActivityResponse: + op = scd_api.OPERATIONS[scd_api.OperationID.DeleteFlight] + url = op.path.format(flight_id=flight_id) + query = query_and_describe(self._session, op.verb, url, scope=self.SCD_SCOPE) + if query.status_code != 200: + raise PlanningActivityError( + f"Attempt to delete flight returned status {query.status_code} rather than 200 as expected", + query, + ) + try: + resp: scd_api.DeleteFlightResponse = ImplicitDict.parse( + query.response.json, scd_api.DeleteFlightResponse + ) + except ValueError as e: + raise PlanningActivityError( + f"Response to delete flight could not be parsed: {str(e)}", query + ) + + old_state = ( + self._plan_statuses[flight_id] + if flight_id in self._plan_statuses + else FlightPlanStatus.NotPlanned + ) + response = PlanningActivityResponse( + flight_id=flight_id, + queries=[query], + activity_result={ + scd_api.DeleteFlightResponseResult.Closed: PlanningActivityResult.Completed, + scd_api.DeleteFlightResponseResult.Failed: PlanningActivityResult.Failed, + }[resp.result], + flight_plan_status={ + scd_api.DeleteFlightResponseResult.Closed: FlightPlanStatus.Closed, + scd_api.DeleteFlightResponseResult.Failed: old_state, + }[resp.result], + ) + if resp.result == scd_api.DeleteFlightResponseResult.Closed: + del self._plan_statuses[flight_id] + else: + self._plan_statuses[flight_id] = response.flight_plan_status + return response + + def report_readiness(self) -> TestPreparationActivityResponse: + op = scd_api.OPERATIONS[scd_api.OperationID.GetStatus] + query = query_and_describe( + self._session, op.verb, op.path, scope=self.SCD_SCOPE + ) + if query.status_code != 200: + raise PlanningActivityError( + f"Attempt to get interface status returned status {query.status_code} rather than 200 as expected", + query, + ) + try: + resp: scd_api.StatusResponse = ImplicitDict.parse( + query.response.json, scd_api.StatusResponse + ) + except ValueError as e: + raise PlanningActivityError( + f"Response to get interface status could not be parsed: {str(e)}", query + ) + + if resp.status == scd_api.StatusResponseStatus.Ready: + errors = [] + elif resp.status == scd_api.StatusResponseStatus.Starting: + errors = ["SCD flight planning interface is still starting (not ready)"] + else: + errors = [f"Unrecognized status '{resp.status}'"] + + # Note that checking capabilities is not included because the SCD flight planning interface is deprecated and does not warrant full support + + return TestPreparationActivityResponse(errors=errors, queries=[query]) + + def clear_area(self, area: Volume4D) -> TestPreparationActivityResponse: + req = scd_api.ClearAreaRequest( + request_id=str(uuid.uuid4()), extent=area.to_interuss_scd_api() + ) + + op = scd_api.OPERATIONS[scd_api.OperationID.ClearArea] + query = query_and_describe( + self._session, op.verb, op.path, json=req, scope=self.SCD_SCOPE + ) + if query.status_code != 200: + raise PlanningActivityError( + f"Attempt to clear area returned status {query.status_code} rather than 200 as expected", + query, + ) + try: + resp: scd_api.ClearAreaResponse = ImplicitDict.parse( + query.response.json, scd_api.ClearAreaResponse + ) + except ValueError as e: + raise PlanningActivityError( + f"Response to clear area could not be parsed: {str(e)}", query + ) + + if resp.outcome.success: + errors = None + else: + errors = [f"[{resp.outcome.timestamp}]: {resp.outcome.message}"] + + return TestPreparationActivityResponse(errors=errors, queries=[query]) diff --git a/monitoring/monitorlib/clients/flight_planning/flight_info.py b/monitoring/monitorlib/clients/flight_planning/flight_info.py new file mode 100644 index 0000000000..2dfb838bc0 --- /dev/null +++ b/monitoring/monitorlib/clients/flight_planning/flight_info.py @@ -0,0 +1,236 @@ +from enum import Enum +from typing import Optional, List + +from implicitdict import ImplicitDict +from uas_standards.ansi_cta_2063_a import SerialNumber +from uas_standards.en4709_02 import OperatorRegistrationNumber + +from monitoring.monitorlib.geotemporal import Volume4D + + +# ===== ASTM F3548-21 ===== + + +Priority = int +"""Ordinal priority that the flight's operational intent should be assigned, as defined in ASTM F3548-21.""" + + +class ASTMF354821OpIntentInformation(ImplicitDict): + """Information provided about a flight plan that is necessary for ASTM F3548-21.""" + + priority: Optional[Priority] + + +# ===== U-space ===== + + +class FlightAuthorisationDataOperationCategory(str, Enum): + """Category of UAS operation (‘open’, ‘specific’, ‘certified’) as defined in COMMISSION DELEGATED REGULATION (EU) 2019/945. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.""" + + Unknown = "Unknown" + Open = "Open" + Specific = "Specific" + Certified = "Certified" + + +class OperationMode(str, Enum): + """Specify if the operation is a `VLOS` or `BVLOS` operation. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 2.""" + + Undeclared = "Undeclared" + Vlos = "Vlos" + Bvlos = "Bvlos" + + +class UASClass(str, Enum): + """Specify the class of the UAS to be flown, the specifition matches EASA class identification label categories. UAS aircraft class as defined in COMMISSION DELEGATED REGULATION (EU) 2019/945 (C0 to C4) and COMMISSION DELEGATED REGULATION (EU) 2020/1058 (C5 and C6). This field is required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.""" + + Other = "Other" + C0 = "C0" + C1 = "C1" + C2 = "C2" + C3 = "C3" + C4 = "C4" + C5 = "C5" + C6 = "C6" + + +class FlightAuthorisationData(ImplicitDict): + """The details of a UAS flight authorization request, as received from the user. + + Note that a full description of a flight authorisation must include mandatory information required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664 for an UAS flight authorisation request. Reference: https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:32021R0664&from=EN#d1e32-178-1 + """ + + uas_serial_number: SerialNumber + """Unique serial number of the unmanned aircraft or, if the unmanned aircraft is privately built, the unique serial number of the add-on. This is expressed in the ANSI/CTA-2063 Physical Serial Number format. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 1.""" + + operation_mode: OperationMode + + operation_category: FlightAuthorisationDataOperationCategory + """Category of UAS operation (‘open’, ‘specific’, ‘certified’) as defined in COMMISSION DELEGATED REGULATION (EU) 2019/945. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.""" + + uas_class: UASClass + + identification_technologies: List[str] + """Technology used to identify the UAS. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 6.""" + + uas_type_certificate: Optional[str] + """Provisional field. Not applicable as of September 2021. Required only if `uas_class` is set to `other` by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 4.""" + + connectivity_methods: List[str] + """Connectivity methods. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 7.""" + + endurance_minutes: int + """Endurance of the UAS. This is expressed in minutes. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 8.""" + + emergency_procedure_url: str + """The URL at which the applicable emergency procedure in case of a loss of command and control link may be retrieved. Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 9.""" + + operator_id: OperatorRegistrationNumber + """Registration number of the UAS operator. + The format is defined in EASA Easy Access Rules for Unmanned Aircraft Systems GM1 to AMC1 + Article 14(6) Registration of UAS operators and ‘certified’ UAS. + Required by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 10. + """ + + uas_id: Optional[str] + """When applicable, the registration number of the unmanned aircraft. + This is expressed using the nationality and registration mark of the unmanned aircraft in + line with ICAO Annex 7. + Specified by ANNEX IV of COMMISSION IMPLEMENTING REGULATION (EU) 2021/664, paragraph 10. + """ + + +# ===== RPAS Operating Rules 2.6 ===== + + +class RPAS26FlightDetailsOperatorType(str, Enum): + """The type of operator.""" + + Recreational = "Recreational" + CommercialExcluded = "CommercialExcluded" + ReOC = "ReOC" + + +class RPAS26FlightDetailsAircraftType(str, Enum): + """Type of vehicle being used as per ASTM F3411-22a.""" + + NotDeclared = "NotDeclared" + Aeroplane = "Aeroplane" + Helicopter = "Helicopter" + Gyroplane = "Gyroplane" + HybridLift = "HybridLift" + Ornithopter = "Ornithopter" + Glider = "Glider" + Kite = "Kite" + FreeBalloon = "FreeBalloon" + CaptiveBalloon = "CaptiveBalloon" + Airship = "Airship" + FreeFallOrParachute = "FreeFallOrParachute" + Rocket = "Rocket" + TetheredPoweredAircraft = "TetheredPoweredAircraft" + GroundObstacle = "GroundObstacle" + Other = "Other" + + +class RPAS26FlightDetailsFlightProfile(str, Enum): + """Type of flight profile.""" + + AutomatedGrid = "AutomatedGrid" + AutomatedWaypoint = "AutomatedWaypoint" + Manual = "Manual" + + +class RPAS26FlightDetails(ImplicitDict): + """Information about a flight necessary to plan successfully using the RPAS Platform Operating Rules version 2.6.""" + + operator_type: Optional[RPAS26FlightDetailsOperatorType] + """The type of operator.""" + + uas_serial_numbers: Optional[List[str]] + """The list of UAS/drone serial numbers that will be operated during the operation.""" + + uas_registration_numbers: Optional[List[str]] + """The list of UAS/drone registration numbers that will be operated during the operation.""" + + aircraft_type: Optional[RPAS26FlightDetailsAircraftType] + """Type of vehicle being used as per ASTM F3411-22a.""" + + flight_profile: Optional[RPAS26FlightDetailsFlightProfile] + """Type of flight profile.""" + + pilot_license_number: Optional[str] + """License number for the pilot.""" + + pilot_phone_number: Optional[str] + """Contact phone number for the pilot.""" + + operator_number: Optional[str] + """Operator number.""" + + +# ===== General flight information ===== + + +FlightID = str + + +class AirspaceUsageState(str, Enum): + """User's current usage of the airspace defined in the flight plan.""" + + Planned = "Planned" + """The user intends to fly according to this flight plan, but is not currently using the defined area with an active UAS.""" + + InUse = "InUse" + """The user is currently using the defined area with an active UAS.""" + + +class UasState(str, Enum): + """State of the user's UAS associated with a flight plan.""" + + Nominal = "Nominal" + """The user or UAS reports or implies that it is performing nominally.""" + + OffNominal = "OffNominal" + """The user or UAS reports or implies that it is temporarily not performing nominally, but expects to be able to recover to normal operation.""" + + Contingent = "Contingent" + """The user or UAS reports or implies that it is not performing nominally and may be unable to recover to normal operation.""" + + +class BasicFlightPlanInformation(ImplicitDict): + """Basic information about a flight plan that an operator and/or UAS can be expected to provide in most flight planning scenarios.""" + + usage_state: AirspaceUsageState + """User's current usage of the airspace specified in the flight plan.""" + + uas_state: UasState + """State of the user's UAS associated with this flight plan.""" + + area: List[Volume4D] + """User intends to or may fly anywhere in this entire area.""" + + +class FlightInfo(ImplicitDict): + """Details of user's intent to create or modify a flight plan.""" + + basic_information: BasicFlightPlanInformation + + astm_f3548_21: Optional[ASTMF354821OpIntentInformation] + + uspace_flight_authorisation: Optional[FlightAuthorisationData] + + rpas_operating_rules_2_6: Optional[RPAS26FlightDetails] + + additional_information: Optional[dict] + """Any information relevant to a particular jurisdiction or use case not described in the standard schema. The keys and values must be agreed upon between the test designers and USSs under test.""" + + +class ExecutionStyle(str, Enum): + Hypothetical = "Hypothetical" + """The user does not want the USS to actually perform any action regarding the actual flight plan. Instead, the user would like to know the likely outcome if the action were hypothetically attempted. The response to this request will not refer to an actual flight plan, or an actual state change in an existing flight plan, but rather a hypothetical flight plan or a hypothetical change to an existing flight plan.""" + + IfAllowed = "IfAllowed" + """The user would like to perform the requested action if it is allowed. If the requested action is allowed, the USS should actually perform the action (e.g., actually create a new ASTM F3548-21 operational intent). If the requested action is not allowed, the USS should indicate that the action is Rejected and not perform the action. The response to this request will refer to an actual flight plan when appropriate, and never refer to a hypothetical flight plan or status.""" + + InReality = "InReality" + """The user is communicating an actual state of reality. The USS should consider the user to be actually performing (or attempting to perform) this action, regardless of whether or not the action is allowed under relevant UTM rules.""" diff --git a/monitoring/monitorlib/clients/flight_planning/flight_info_template.py b/monitoring/monitorlib/clients/flight_planning/flight_info_template.py new file mode 100644 index 0000000000..eb8ffd4819 --- /dev/null +++ b/monitoring/monitorlib/clients/flight_planning/flight_info_template.py @@ -0,0 +1,53 @@ +from datetime import datetime +from typing import List, Optional + +from implicitdict import ImplicitDict + +from monitoring.monitorlib.clients.flight_planning.flight_info import ( + AirspaceUsageState, + UasState, + ASTMF354821OpIntentInformation, + FlightAuthorisationData, + RPAS26FlightDetails, + BasicFlightPlanInformation, + FlightInfo, +) +from monitoring.monitorlib.geotemporal import Volume4DTemplate, resolve_volume4d + + +class BasicFlightPlanInformationTemplate(ImplicitDict): + """Template to provide (at runtime) basic information about a flight plan that an operator and/or UAS can be expected to provide in most flight planning scenarios.""" + + usage_state: AirspaceUsageState + """User's current usage of the airspace specified in the flight plan.""" + + uas_state: UasState + """State of the user's UAS associated with this flight plan.""" + + area: List[Volume4DTemplate] + """User intends to or may fly anywhere in this entire area.""" + + def resolve(self, start_of_test: datetime) -> BasicFlightPlanInformation: + kwargs = {k: v for k, v in self.items()} + kwargs["area"] = [resolve_volume4d(t, start_of_test) for t in self.area] + return ImplicitDict.parse(kwargs, BasicFlightPlanInformation) + + +class FlightInfoTemplate(ImplicitDict): + """Template to provide (at runtime) details of user's intent to create or modify a flight plan.""" + + basic_information: BasicFlightPlanInformationTemplate + + astm_f3548_21: Optional[ASTMF354821OpIntentInformation] + + uspace_flight_authorisation: Optional[FlightAuthorisationData] + + rpas_operating_rules_2_6: Optional[RPAS26FlightDetails] + + additional_information: Optional[dict] + """Any information relevant to a particular jurisdiction or use case not described in the standard schema. The keys and values must be agreed upon between the test designers and USSs under test.""" + + def resolve(self, start_of_test: datetime) -> FlightInfo: + kwargs = {k: v for k, v in self.items()} + kwargs["basic_information"] = self.basic_information.resolve(start_of_test) + return ImplicitDict.parse(kwargs, FlightInfo) diff --git a/monitoring/monitorlib/clients/flight_planning/planning.py b/monitoring/monitorlib/clients/flight_planning/planning.py new file mode 100644 index 0000000000..8f473f8575 --- /dev/null +++ b/monitoring/monitorlib/clients/flight_planning/planning.py @@ -0,0 +1,71 @@ +from enum import Enum +from typing import Optional, List + +from implicitdict import ImplicitDict + +from monitoring.monitorlib.clients.flight_planning.flight_info import FlightID +from monitoring.monitorlib.fetch import Query + + +class PlanningActivityResult(str, Enum): + """The result of the flight planning operation.""" + + Completed = "Completed" + """The user's flight plan has been updated according to the situation specified by the user.""" + + Rejected = "Rejected" + """The updates the user requested to their flight plan are not allowed according to the rules under which the flight plan is being managed. The reasons for rejection may include a disallowed conflict with another flight during preflight.""" + + Failed = "Failed" + """The USS was not able to successfully authorize or update the flight plan due to a problem with the USS or a downstream system.""" + + NotSupported = "NotSupported" + """The USS's implementation does not support the attempted interaction. For instance, if the request specified a high-priority flight and the USS does not support management of high-priority flights.""" + + +class FlightPlanStatus(str, Enum): + """Status of a user's flight plan.""" + + NotPlanned = "NotPlanned" + """The USS has not created an authorized flight plan for the user.""" + + Planned = "Planned" + """The USS has created an authorized flight plan for the user, but the user may not yet start flying (even if within the time bounds of the flight plan).""" + + OkToFly = "OkToFly" + """The flight plan is in a state such that it is ok for the user to nominally fly within the bounds (including time) of the flight plan.""" + + OffNominal = "OffNominal" + """The flight plan now reflects the operator's actions, but the flight plan is not in a nominal state (e.g., the USS has placed the ASTM F3548-21 operational intent into one of the Nonconforming or Contingent states).""" + + Closed = "Closed" + """The flight plan was closed successfully by the USS and is now out of the UTM system.""" + + +class AdvisoryInclusion(str, Enum): + """Indication of whether any advisories or conditions were provided to the user along with the result of a flight planning attempt.""" + + Unknown = "Unknown" + """It is unknown or irrelevant whether advisories or conditions were provided to the user.""" + + AtLeastOneAdvisoryOrCondition = "AtLeastOneAdvisoryOrCondition" + """At least one advisory or condition was provided to the user.""" + + NoAdvisoriesOrConditions = "NoAdvisoriesOrConditions" + """No advisories or conditions were provided to the user.""" + + +class PlanningActivityResponse(ImplicitDict): + flight_id: FlightID + """Identity of flight for which the planning activity was conducted.""" + + queries: List[Query] + """Queries used to accomplish this activity.""" + + activity_result: PlanningActivityResult + """The result of the flight planning activity.""" + + flight_plan_status: FlightPlanStatus + """Status of the flight plan following the flight planning activity.""" + + includes_advisories: Optional[AdvisoryInclusion] = AdvisoryInclusion.Unknown diff --git a/monitoring/monitorlib/clients/flight_planning/test_preparation.py b/monitoring/monitorlib/clients/flight_planning/test_preparation.py new file mode 100644 index 0000000000..e88bf6679f --- /dev/null +++ b/monitoring/monitorlib/clients/flight_planning/test_preparation.py @@ -0,0 +1,25 @@ +from typing import Optional, List + +from implicitdict import ImplicitDict + +from monitoring.monitorlib.fetch import Query + + +class ClearAreaOutcome(ImplicitDict): + success: Optional[bool] = False + """True if, and only if, all flight plans in the specified area managed by the USS were canceled and removed.""" + + message: Optional[str] + """If the USS was unable to clear the entire area, this message can provide information on the problem encountered.""" + + +class ClearAreaResponse(ImplicitDict): + outcome: ClearAreaOutcome + + +class TestPreparationActivityResponse(ImplicitDict): + errors: Optional[List[str]] = None + """If any errors occurred during this activity, a list of those errors.""" + + queries: List[Query] + """Queries used to accomplish this activity.""" diff --git a/monitoring/monitorlib/geotemporal.py b/monitoring/monitorlib/geotemporal.py index 3d38b90823..ec7d06a8b1 100644 --- a/monitoring/monitorlib/geotemporal.py +++ b/monitoring/monitorlib/geotemporal.py @@ -404,6 +404,26 @@ def resolve_volume4d(template: Volume4DTemplate, start_of_test: datetime) -> Vol class Volume4DCollection(ImplicitDict): volumes: List[Volume4D] + def __add__(self, other): + if isinstance(other, Volume4D): + return Volume4DCollection(volumes=self.volumes + [other]) + elif isinstance(other, Volume4DCollection): + return Volume4DCollection(volumes=self.volumes + other.volumes) + else: + raise NotImplementedError( + f"Cannot add {type(other).__name__} to {type(self).__name__}" + ) + + def __iadd__(self, other): + if isinstance(other, Volume4D): + self.volumes.append(other) + elif isinstance(other, Volume4DCollection): + self.volumes.extend(other.volumes) + else: + raise NotImplementedError( + f"Cannot iadd {type(other).__name__} to {type(self).__name__}" + ) + @property def time_start(self) -> Optional[Time]: return ( @@ -547,3 +567,6 @@ def from_interuss_scd_api( def to_f3548v21(self) -> List[f3548v21.Volume4D]: return [v.to_f3548v21() for v in self.volumes] + + def to_interuss_scd_api(self) -> List[interuss_scd_api.Volume4D]: + return [v.to_interuss_scd_api() for v in self.volumes] diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py index 7b57506605..fae5d9eb01 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_planner.py @@ -5,8 +5,25 @@ from implicitdict import ImplicitDict from monitoring.monitorlib import infrastructure, fetch +from monitoring.monitorlib.clients.flight_planning.client import PlanningActivityError +from monitoring.monitorlib.clients.flight_planning.client_scd import ( + SCDFlightPlannerClient, +) +from monitoring.monitorlib.clients.flight_planning.flight_info import ( + ExecutionStyle, + FlightInfo, + BasicFlightPlanInformation, + ASTMF354821OpIntentInformation, + FlightAuthorisationData, + AirspaceUsageState, + UasState, +) +from monitoring.monitorlib.clients.flight_planning.planning import ( + PlanningActivityResult, + FlightPlanStatus, +) from monitoring.monitorlib.fetch import QueryError, Query -from monitoring.monitorlib.geotemporal import Volume4D +from monitoring.monitorlib.geotemporal import Volume4D, Volume4DCollection from uas_standards.interuss.automated_testing.scd.v1.api import ( InjectFlightResponseResult, DeleteFlightResponseResult, @@ -15,6 +32,8 @@ InjectFlightRequest, ClearAreaResponse, ClearAreaRequest, + OperationalIntentState, + ClearAreaOutcome, ) from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( SCOPE_SCD_QUALIFIER_INJECT, @@ -45,7 +64,9 @@ def __init__(self, *args, **kwargs): class FlightPlanner: - """Manages the state and the interactions with flight planner USS""" + """Manages the state and the interactions with flight planner USS. + + Note: this class will be deprecated in favor of FlightPlannerClient.""" def __init__( self, @@ -53,9 +74,10 @@ def __init__( auth_adapter: infrastructure.AuthAdapter, ): self.config = config - self.client = infrastructure.UTMClientSession( + session = infrastructure.UTMClientSession( self.config.injection_base_url, auth_adapter, config.timeout_seconds ) + self.scd_client = SCDFlightPlannerClient(session) # Flights injected by this target. self.created_flight_ids: Set[str] = set() @@ -78,114 +100,137 @@ def request_flight( request: InjectFlightRequest, flight_id: Optional[str] = None, ) -> Tuple[InjectFlightResponse, fetch.Query, str]: - if not flight_id: - flight_id = str(uuid.uuid4()) - url = "{}/v1/flights/{}".format(self.config.injection_base_url, flight_id) - - query = fetch.query_and_describe( - self.client, - "PUT", - url, - json=request, - scope=SCOPE_SCD_QUALIFIER_INJECT, - server_id=self.config.participant_id, - ) - if query.status_code != 200: - raise QueryError( - f"Inject flight query to {url} returned {query.status_code}", [query] - ) - try: - result = ImplicitDict.parse( - query.response.get("json", {}), InjectFlightResponse - ) - except ValueError as e: - raise QueryError( - f"Inject flight response from {url} could not be decoded: {str(e)}", - [query], + usage_states = { + OperationalIntentState.Accepted: AirspaceUsageState.Planned, + OperationalIntentState.Activated: AirspaceUsageState.InUse, + OperationalIntentState.Nonconforming: AirspaceUsageState.InUse, + OperationalIntentState.Contingent: AirspaceUsageState.InUse, + } + uas_states = { + OperationalIntentState.Accepted: UasState.Nominal, + OperationalIntentState.Activated: UasState.Nominal, + OperationalIntentState.Nonconforming: UasState.OffNominal, + OperationalIntentState.Contingent: UasState.Contingent, + } + if ( + request.operational_intent.state + in (OperationalIntentState.Accepted, OperationalIntentState.Activated) + and request.operational_intent.off_nominal_volumes + ): + # This invalid request can no longer be represented with a standard flight planning request; reject it at the client level instead + raise ValueError( + f"Request for nominal {request.operational_intent.state} operational intent is invalid because it contains off-nominal volumes" ) + v4c = Volume4DCollection.from_interuss_scd_api( + request.operational_intent.volumes + ) + Volume4DCollection.from_interuss_scd_api( + request.operational_intent.off_nominal_volumes + ) + basic_information = BasicFlightPlanInformation( + usage_state=usage_states[request.operational_intent.state], + uas_state=uas_states[request.operational_intent.state], + area=v4c.volumes, + ) + astm_f3548v21 = ASTMF354821OpIntentInformation( + priority=request.operational_intent.priority + ) + uspace_flight_authorisation = ImplicitDict.parse( + request.flight_authorisation, FlightAuthorisationData + ) + flight_info = FlightInfo( + basic_information=basic_information, + astm_f3548_21=astm_f3548v21, + uspace_flight_authorisation=uspace_flight_authorisation, + ) - if result.result == InjectFlightResponseResult.Planned: + if not flight_id: + try: + resp = self.scd_client.try_plan_flight( + flight_info, ExecutionStyle.IfAllowed + ) + except PlanningActivityError as e: + raise QueryError(str(e), e.queries) + flight_id = resp.flight_id + else: + try: + resp = self.scd_client.try_update_flight( + flight_id, flight_info, ExecutionStyle.IfAllowed + ) + except PlanningActivityError as e: + raise QueryError(str(e), e.queries) + + if resp.activity_result == PlanningActivityResult.Failed: + result = InjectFlightResponseResult.Failed + elif resp.activity_result == PlanningActivityResult.NotSupported: + result = InjectFlightResponseResult.NotSupported + elif resp.activity_result == PlanningActivityResult.Rejected: + result = InjectFlightResponseResult.Rejected + elif resp.activity_result == PlanningActivityResult.Completed: + if resp.flight_plan_status == FlightPlanStatus.Planned: + result = InjectFlightResponseResult.Planned + elif resp.flight_plan_status == FlightPlanStatus.OkToFly: + result = InjectFlightResponseResult.ReadyToFly + elif resp.flight_plan_status == FlightPlanStatus.OffNominal: + result = InjectFlightResponseResult.ReadyToFly + else: + raise NotImplementedError( + f"Unable to handle '{resp.flight_plan_status}' FlightPlanStatus with {resp.activity_result} PlanningActivityResult" + ) self.created_flight_ids.add(flight_id) + else: + raise NotImplementedError( + f"Unable to handle '{resp.activity_result}' PlanningActivityResult" + ) + + response = InjectFlightResponse( + result=result, + operational_intent_id="", + ) - return result, query, flight_id + return response, resp.queries[0], flight_id def cleanup_flight( self, flight_id: str ) -> Tuple[DeleteFlightResponse, fetch.Query]: - url = "{}/v1/flights/{}".format(self.config.injection_base_url, flight_id) - query = fetch.query_and_describe( - self.client, - "DELETE", - url, - scope=SCOPE_SCD_QUALIFIER_INJECT, - server_id=self.config.participant_id, - ) - if query.status_code != 200: - raise QueryError( - f"Delete flight query to {url} returned {query.status_code}", [query] - ) try: - result = ImplicitDict.parse( - query.response.get("json", {}), DeleteFlightResponse - ) - except ValueError as e: - raise QueryError( - f"Delete flight response from {url} could not be decoded: {str(e)}", - [query], - ) - - if result.result == DeleteFlightResponseResult.Closed: + resp = self.scd_client.try_end_flight(flight_id, ExecutionStyle.IfAllowed) + except PlanningActivityError as e: + raise QueryError(str(e), e.queries) + + if ( + resp.activity_result == PlanningActivityResult.Completed + and resp.flight_plan_status == FlightPlanStatus.Closed + ): self.created_flight_ids.remove(flight_id) - return result, query - - def get_readiness(self) -> Tuple[Optional[str], Query]: - url_status = "{}/v1/status".format(self.config.injection_base_url) - version_query = fetch.query_and_describe( - self.client, - "GET", - url_status, - scope=SCOPE_SCD_QUALIFIER_INJECT, - server_id=self.config.participant_id, - ) - if version_query.status_code != 200: return ( - f"Status query to {url_status} returned {version_query.status_code}", - version_query, + DeleteFlightResponse(result=DeleteFlightResponseResult.Closed), + resp.queries[0], ) - try: - ImplicitDict.parse(version_query.response.get("json", {}), StatusResponse) - except ValueError as e: + else: return ( - f"Status response from {url_status} could not be decoded: {str(e)}", - version_query, + DeleteFlightResponse(result=DeleteFlightResponseResult.Failed), + resp.queries[0], ) - return None, version_query + def get_readiness(self) -> Tuple[Optional[str], Query]: + try: + resp = self.scd_client.report_readiness() + except PlanningActivityError as e: + return str(e), e.queries[0] + return None, resp.queries[0] def clear_area(self, extent: Volume4D) -> Tuple[ClearAreaResponse, fetch.Query]: - req = ClearAreaRequest( - request_id=str(uuid.uuid4()), extent=extent.to_f3548v21() - ) - url = f"{self.config.injection_base_url}/v1/clear_area_requests" - query = fetch.query_and_describe( - self.client, - "POST", - url, - scope=SCOPE_SCD_QUALIFIER_INJECT, - json=req, - server_id=self.config.participant_id, - ) - if query.status_code != 200: - raise QueryError( - f"Clear area query to {url} returned {query.status_code}", [query] - ) try: - result = ImplicitDict.parse( - query.response.get("json", {}), ClearAreaResponse - ) - except ValueError as e: - raise QueryError( - f"Clear area response from {url} could not be decoded: {str(e)}", - [query], - ) - return result, query + resp = self.scd_client.clear_area(extent) + except PlanningActivityError as e: + raise QueryError(str(e), e.queries) + success = False if resp.errors else True + return ( + ClearAreaResponse( + outcome=ClearAreaOutcome( + success=success, + timestamp=resp.queries[0].response.reported, + ) + ), + resp.queries[0], + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md index 9d572c3d46..dc390d82e4 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md @@ -14,8 +14,6 @@ Notably the following requirements: FlightIntentsResource that provides the following flight intents: - `valid_flight`: a valid operational intent upon which other invalid ones are derived, in `Accepted` state - `valid_activated`: state mutation `Activated` - - `invalid_accepted_offnominal`: off-nominal volumes mutation: has (any valid) off-nominal volume - - `invalid_activated_offnominal`: state mutation `Activated` - `invalid_too_far_away`: reference time mutation: reference time pulled back so that it is like the operational intent is attempted to be planned more than OiMaxPlanHorizon = 30 days ahead of time - `valid_conflict_tiny_overlap`: volumes mutation: has a volume that overlaps with `valid_op_intent` just above IntersectionMinimumPrecision = 1cm in a way that must result as a conflict @@ -55,69 +53,6 @@ to reject or accept the flight. If the USS indicates that the injection attempt ### [Validate flight intent too far ahead of time not planned test step](../validate_not_shared_operational_intent.md) -## Attempt to specify off-nominal volume in Accepted and Activated states test case -### Attempt to plan flight with an off-nominal volume test step -The user flight intent that the test driver attempts to plan has an off-nominal volume. -As such, the planning attempt should be rejected. - -#### Incorrectly planned check -If the USS successfully plans the flight or otherwise fails to indicate a rejection, it means that it failed to validate -the intent provided. Therefore, this check will fail if the USS indicates success in creating the flight from the user -flight intent, per **[astm.f3548.v21.OPIN0015](../../../../requirements/astm/f3548/v21.md)**. - -#### Failure check -All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS -to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. - -### [Validate flight intent with an off-nominal volume not planned test step](../validate_not_shared_operational_intent.md) - -### [Plan valid flight intent test step](../../../flight_planning/plan_flight_intent.md) -The valid flight intent should be successfully planned by the flight planner. - -### [Validate flight sharing test step](../validate_shared_operational_intent.md) - -### Attempt to modify planned flight with an off-nominal volume test step -The user flight intent that the test driver attempts to modify has an off-nominal volume and is in the `Accepted` state. -As such, the modification attempt should be rejected. - -#### Incorrectly modified check -If the USS successfully modifies the flight or otherwise fails to indicate a rejection, it means that it failed to validate -the intent provided. Therefore, this check will fail if the USS indicates success in modifying the flight from the user -flight intent, per **[astm.f3548.v21.OPIN0015](../../../../requirements/astm/f3548/v21.md)**. - -#### Failure check -All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS -to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. - -### [Validate planned flight not modified test step](../validate_shared_operational_intent.md) -Validate that the planned flight intent was not modified with an off-nominal volume. - -### [Activate valid flight intent test step](../../../flight_planning/activate_flight_intent.md) -The valid flight intent should be successfully activated by the flight planner. - -### [Validate flight sharing test step](../validate_shared_operational_intent.md) - -### Attempt to modify activated flight with an off-nominal volume test step -The user flight intent that the test driver attempts to modify has an off-nominal volume and is in the `Activated` state. -As such, the modification attempt should be rejected. - -#### Incorrectly modified check -If the USS successfully modifies the flight or otherwise fails to indicate a rejection, it means that it failed to validate -the intent provided. Therefore, this check will fail if the USS indicates success in modifying the flight from the user -flight intent, per **[astm.f3548.v21.OPIN0015](../../../../requirements/astm/f3548/v21.md)**. - -#### Failure check -All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS -to reject or accept the flight. If the USS indicates that the injection attempt failed, this check will fail per -**[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. - -### [Validate activated flight not modified test step](../validate_shared_operational_intent.md) -Validate that the activated flight intent was not modified with an off-nominal volume. - -### [Delete valid flight intent test step](../../../flight_planning/delete_flight_intent.md) - ## Validate transition to Ended state after cancellation test case ### [Plan flight intent test step](../../../flight_planning/plan_flight_intent.md) The valid flight intent should be successfully planned by the flight planner. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index 226cb91a99..a369a9a7e1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -39,9 +39,6 @@ class FlightIntentValidation(TestScenario): valid_flight: FlightIntent valid_activated: FlightIntent - invalid_accepted_offnominal: FlightIntent - invalid_activated_offnominal: FlightIntent - invalid_too_far_away: FlightIntent valid_conflict_tiny_overlap: FlightIntent @@ -74,15 +71,11 @@ def __init__( self.valid_flight, self.valid_activated, self.invalid_too_far_away, - self.invalid_accepted_offnominal, - self.invalid_activated_offnominal, self.valid_conflict_tiny_overlap, ) = ( flight_intents["valid_flight"], flight_intents["valid_activated"], flight_intents["invalid_too_far_away"], - flight_intents["invalid_accepted_offnominal"], - flight_intents["invalid_activated_offnominal"], flight_intents["valid_conflict_tiny_overlap"], ) @@ -98,14 +91,6 @@ def __init__( self.invalid_too_far_away.request.operational_intent.state == OperationalIntentState.Accepted ), "invalid_too_far_away must have state Accepted" - assert ( - self.invalid_accepted_offnominal.request.operational_intent.state - == OperationalIntentState.Accepted - ), "invalid_accepted_offnominal must have state Accepted" - assert ( - self.invalid_activated_offnominal.request.operational_intent.state - == OperationalIntentState.Activated - ), "invalid_activated_offnominal must have state Activated" assert ( self.valid_conflict_tiny_overlap.request.operational_intent.state == OperationalIntentState.Accepted @@ -121,19 +106,6 @@ def __init__( time_delta.days > OiMaxPlanHorizonDays ), f"invalid_too_far_away must have start time more than {OiMaxPlanHorizonDays} days ahead of reference time, got {time_delta}" - assert ( - len( - self.invalid_accepted_offnominal.request.operational_intent.off_nominal_volumes - ) - > 0 - ), "invalid_accepted_offnominal must have at least one off-nominal volume" - assert ( - len( - self.invalid_activated_offnominal.request.operational_intent.off_nominal_volumes - ) - > 0 - ), "invalid_activated_offnominal must have at least one off-nominal volume" - assert Volume4DCollection.from_interuss_scd_api( self.valid_flight.request.operational_intent.volumes ).intersects_vol4s( @@ -167,12 +139,6 @@ def run(self): self._attempt_invalid() self.end_test_case() - self.begin_test_case( - "Attempt to specify off-nominal volume in Accepted and Activated states" - ) - self._attempt_invalid_offnominal() - self.end_test_case() - self.begin_test_case("Validate transition to Ended state after cancellation") self._validate_ended_cancellation() self.end_test_case() @@ -207,8 +173,6 @@ def _setup(self) -> bool: [ self.valid_flight, self.valid_activated, - self.invalid_accepted_offnominal, - self.invalid_activated_offnominal, self.invalid_too_far_away, self.valid_conflict_tiny_overlap, ], @@ -236,105 +200,6 @@ def _attempt_invalid(self): ) validator.expect_not_shared() - def _attempt_invalid_offnominal(self): - with OpIntentValidator( - self, - self.tested_uss, - self.dss, - "Validate flight intent with an off-nominal volume not planned", - self._intents_extent, - ) as validator: - submit_flight_intent( - self, - "Attempt to plan flight with an off-nominal volume", - "Incorrectly planned", - {InjectFlightResponseResult.Rejected}, - {InjectFlightResponseResult.Failed: "Failure"}, - self.tested_uss, - self.invalid_accepted_offnominal.request, - ) - validator.expect_not_shared() - - with OpIntentValidator( - self, - self.tested_uss, - self.dss, - "Validate flight sharing", - self._intents_extent, - ) as validator: - _, valid_flight_id = plan_flight_intent( - self, - "Plan valid flight intent", - self.tested_uss, - self.valid_flight.request, - ) - valid_flight_op_intent_ref = validator.expect_shared( - self.valid_flight.request - ) - - with OpIntentValidator( - self, - self.tested_uss, - self.dss, - "Validate planned flight not modified", - self._intents_extent, - valid_flight_op_intent_ref, - ) as validator: - submit_flight_intent( - self, - "Attempt to modify planned flight with an off-nominal volume", - "Incorrectly modified", - {InjectFlightResponseResult.Rejected}, - {InjectFlightResponseResult.Failed: "Failure"}, - self.tested_uss, - self.invalid_accepted_offnominal.request, - ) - valid_flight_op_intent_ref = validator.expect_shared( - self.valid_flight.request - ) - - with OpIntentValidator( - self, - self.tested_uss, - self.dss, - "Validate flight sharing", - self._intents_extent, - valid_flight_op_intent_ref, - ) as validator: - activate_flight_intent( - self, - "Activate valid flight intent", - self.tested_uss, - self.valid_activated.request, - valid_flight_id, - ) - valid_flight_op_intent_ref = validator.expect_shared( - self.valid_activated.request - ) - - with OpIntentValidator( - self, - self.tested_uss, - self.dss, - "Validate activated flight not modified", - self._intents_extent, - valid_flight_op_intent_ref, - ) as validator: - submit_flight_intent( - self, - "Attempt to modify activated flight with an off-nominal volume", - "Incorrectly modified", - {InjectFlightResponseResult.Rejected}, - {InjectFlightResponseResult.Failed: "Failure"}, - self.tested_uss, - self.invalid_activated_offnominal.request, - ) - validator.expect_shared(self.valid_flight.request) - - _ = delete_flight_intent( - self, "Delete valid flight intent", self.tested_uss, valid_flight_id - ) - def _validate_ended_cancellation(self): with OpIntentValidator( self, @@ -382,7 +247,10 @@ def _validate_precision_intersection(self): self, "Attempt to plan flight conflicting by a tiny overlap", "Incorrectly planned", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, self.tested_uss, self.valid_conflict_tiny_overlap.request, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index 938f66ec90..4f71ae140d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -455,10 +455,6 @@ def _modify_activated_flight_preexisting_conflict( self.flight_1_activated_time_range_A.request ) - # TODO: the following call requires the control USS to support CMSA role, - # but as there is currently no explicit way of knowing if it is the case - # or not, we assume that a Rejected result means the USS does not - # support the CMSA role, in which case we interrupt the scenario. with OpIntentValidator( self, self.control_uss, @@ -473,15 +469,15 @@ def _modify_activated_flight_preexisting_conflict( "Successful transition to non-conforming state", { InjectFlightResponseResult.Planned, - InjectFlightResponseResult.Rejected, + InjectFlightResponseResult.NotSupported, }, {InjectFlightResponseResult.Failed: "Failure"}, self.control_uss, self.flight_2_equal_prio_nonconforming_time_range_A.request, self.flight_2_id, ) - if resp_flight_2.result == InjectFlightResponseResult.Rejected: - msg = f"{self.control_uss.config.participant_id} rejected transition to a Nonconforming state because it does not support CMSA role, execution of the scenario was stopped without failure" + if resp_flight_2.result == InjectFlightResponseResult.NotSupported: + msg = f"{self.control_uss.config.participant_id} does not support the transition to a Nonconforming state; execution of the scenario was stopped without failure" self.record_note("Control USS does not support CMSA role", msg) raise ScenarioCannotContinueError(msg) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md index b6fac6fae6..249b93b094 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/validate_shared_operational_intent.md @@ -30,7 +30,7 @@ If the operational intent details reported by the USS do not match the user's fl ## Off-nominal volumes check -**astm.f3548.v21.OPIN0015** +**astm.f3548.v21.OPIN0015** specifies that nominal operational intents (Accepted and Activated) must not include any off-nominal 4D volumes, so this check will fail if an Accepted or Activated operational intent includes off-nominal volumes. ## Vertices check diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py index 01a1f6ab61..76271c3092 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/prioritization_test_steps.py @@ -38,7 +38,10 @@ def plan_priority_conflict_flight_intent( scenario, test_step, "Incorrectly planned", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -67,7 +70,10 @@ def modify_planned_priority_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -97,7 +103,10 @@ def activate_priority_conflict_flight_intent( scenario, test_step, "Incorrectly activated", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -127,7 +136,10 @@ def modify_activated_priority_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -156,7 +168,10 @@ def plan_conflict_flight_intent( scenario, test_step, "Incorrectly planned", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -185,7 +200,10 @@ def modify_planned_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -215,7 +233,10 @@ def activate_conflict_flight_intent( scenario, test_step, "Incorrectly activated", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, @@ -245,7 +266,10 @@ def modify_activated_conflict_flight_intent( scenario, test_step, "Incorrectly modified", - {InjectFlightResponseResult.ConflictWithFlight}, + { + InjectFlightResponseResult.ConflictWithFlight, + InjectFlightResponseResult.Rejected, + }, {InjectFlightResponseResult.Failed: "Failure"}, flight_planner, flight_intent, diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py index 502a7f2eff..36d742f56a 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/test_steps.py @@ -65,7 +65,9 @@ def clear_area( check.record_failed( summary="Area could not be cleared", severity=Severity.High, - details=f'Participant indicated "{resp.outcome.message}"', + details=f'Participant indicated "{resp.outcome.message}"' + if "message" in resp.outcome + else "See query", query_timestamps=[query.request.timestamp], ) @@ -385,7 +387,9 @@ def cleanup_flights( else: check.record_failed( summary="Failed to delete flight", - details=f"USS indicated: {resp.notes}", + details=f"USS indicated: {resp.notes}" + if "notes" in resp + else "See query", severity=Severity.Medium, query_timestamps=[query.request.timestamp], )