Skip to content

Commit

Permalink
Merge branch 'main' into translate-to
Browse files Browse the repository at this point in the history
  • Loading branch information
BenjaminPelletier committed Dec 15, 2023
2 parents dcae0c3 + 2bbe9cb commit 343ac20
Show file tree
Hide file tree
Showing 8 changed files with 135 additions and 25 deletions.
38 changes: 22 additions & 16 deletions monitoring/uss_qualifier/resources/astm/f3548/v21/dss.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
UssAvailabilityState,
UssAvailabilityStatusResponse,
GetOperationalIntentReferenceResponse,
OPERATIONS,
OperationID,
)

# A base URL for a USS that is not expected to be ever called
Expand Down Expand Up @@ -77,12 +79,12 @@ def __init__(
def find_op_intent(
self, extent: Volume4D
) -> Tuple[List[OperationalIntentReference], fetch.Query]:
url = "/dss/v1/operational_intent_references/query"
op = OPERATIONS[OperationID.QueryOperationalIntentReferences]
req = QueryOperationalIntentReferenceParameters(area_of_interest=extent)
query = fetch.query_and_describe(
self.client,
"POST",
url,
op.verb,
op.path,
QueryType.F3548v21DSSQueryOperationalIntentReferences,
self.participant_id,
scope=SCOPE_SC,
Expand All @@ -103,11 +105,11 @@ def get_op_intent_reference(
"""
Retrieve an OP Intent from the DSS, using only its ID
"""
url = f"/dss/v1/operational_intent_references/{op_intent_id}"
op = OPERATIONS[OperationID.GetOperationalIntentReference]
query = fetch.query_and_describe(
self.client,
"GET",
url,
op.verb,
op.path.format(entityid=op_intent_id),
QueryType.F3548v21DSSGetOperationalIntentReference,
self.participant_id,
scope=SCOPE_SC,
Expand All @@ -125,11 +127,11 @@ def get_full_op_intent(
op_intent_ref: OperationalIntentReference,
uss_participant_id: Optional[str] = None,
) -> Tuple[OperationalIntent, fetch.Query]:
url = f"{op_intent_ref.uss_base_url}/uss/v1/operational_intents/{op_intent_ref.id}"
op = OPERATIONS[OperationID.GetOperationalIntentDetails]
query = fetch.query_and_describe(
self.client,
"GET",
url,
op.verb,
f"{op_intent_ref.uss_base_url}{op.path.format(entityid=op_intent_ref.id)}",
QueryType.F3548v21USSGetOperationalIntentDetails,
uss_participant_id,
scope=SCOPE_SC,
Expand Down Expand Up @@ -157,10 +159,12 @@ def put_op_intent(
]:
oi_uuid = str(uuid.uuid4()) if id is None else id
if ovn is None:
url = f"/dss/v1/operational_intent_references/{oi_uuid}"
op = OPERATIONS[OperationID.CreateOperationalIntentReference]
url = op.path.format(entityid=oi_uuid)
query_type = QueryType.F3548v21DSSCreateOperationalIntentReference
else:
url = f"/dss/v1/operational_intent_references/{oi_uuid}/{ovn}"
op = OPERATIONS[OperationID.UpdateOperationalIntentReference]
url = op.path.format(entityid=oi_uuid, ovn=ovn)
query_type = QueryType.F3548v21DSSUpdateOperationalIntentReference

req = PutOperationalIntentReferenceParameters(
Expand All @@ -172,7 +176,7 @@ def put_op_intent(
)
query = fetch.query_and_describe(
self.client,
"PUT",
op.verb,
url,
query_type,
self.participant_id,
Expand All @@ -198,10 +202,11 @@ def delete_op_intent(
Optional[List[SubscriberToNotify]],
fetch.Query,
]:
op = OPERATIONS[OperationID.DeleteOperationalIntentReference]
query = fetch.query_and_describe(
self.client,
"DELETE",
f"/dss/v1/operational_intent_references/{id}/{ovn}",
op.verb,
op.path.format(entityid=id, ovn=ovn),
QueryType.F3548v21DSSDeleteOperationalIntentReference,
self.participant_id,
scope=SCOPE_SC,
Expand Down Expand Up @@ -237,10 +242,11 @@ def set_uss_availability(
old_version=version,
availability=availability,
)
op = OPERATIONS[OperationID.SetUssAvailability]
query = fetch.query_and_describe(
self.client,
"PUT",
f"/dss/v1/uss_availability/{uss_id}",
op.verb,
op.path.format(uss_id=uss_id),
QueryType.F3548v21DSSSetUssAvailability,
self.participant_id,
scope=SCOPE_AA,
Expand Down
14 changes: 14 additions & 0 deletions monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,17 @@ planners provided as resource are used to determine and evaluate the 95th percen

If the 95th percentile of the requests durations is higher than the threshold `MaxRespondToOIDetailsRequest` (1 second),
this check will fail per **[astm.f3548.v21.SCD0075](../../../requirements/astm/f3548/v21.md)**.

## Interoperability test instance is available test case

### Interoperability test instance is available test step

This step verifies that interactions with the interoperability test instances happened and where at least partly successful.

#### Interoperability test instance is available check

This check ensures that interactions with the interoperability test instance that each USS must provide are possible.

If all interactions fail, or if no test instance can be reached, the USS is failing to meet **[astm.f3548.v21.GEN0300](../../../requirements/astm/f3548/v21.md)**.

If no interaction with a test instance was found, this check is skipped.
79 changes: 74 additions & 5 deletions monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,17 @@
from typing import List, Dict

from uas_standards.astm.f3548.v21 import constants

from monitoring.monitorlib import fetch
from monitoring.monitorlib.fetch import evaluation, QueryType
from monitoring.uss_qualifier.common_data_definitions import Severity
from monitoring.uss_qualifier.configurations.configuration import ParticipantID
from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource
from monitoring.uss_qualifier.suites.suite import ExecutionContext

from uas_standards.astm.f3548.v21 import constants

from monitoring.uss_qualifier.scenarios.scenario import TestScenario
from monitoring.uss_qualifier.suites.suite import ExecutionContext


class AggregateChecks(TestScenario):

_queries: List[fetch.Query]
_attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {}

Expand Down Expand Up @@ -76,6 +74,14 @@ def run(self, context: ExecutionContext):
self.end_test_step()
self.end_test_case()

self.begin_test_case("Interoperability test instance is available")
self.begin_test_step("Interoperability test instance is available")

self._confirm_test_harness_queries_work()

self.end_test_step()
self.end_test_case()

self.end_test_scenario()

def _op_intent_details_step(self):
Expand Down Expand Up @@ -118,3 +124,66 @@ def _op_intent_details_step(self):
f"{participant}/{QueryType.F3548v21USSGetOperationalIntentDetails}",
f"checked performances on {len(durations)} queries, 95th percentile: {p95}s",
)

def _confirm_test_harness_queries_work(self):
"""
For each different type of call to the interoperability test instance,
we look for at least one successful query.
"""
for participant, queries_by_type in self._attributed_queries.items():
self._validate_participant_test_interop_instance(
participant, queries_by_type
)

def _validate_participant_test_interop_instance(
self,
participant_id: str,
participant_queries: dict[QueryType, List[fetch.Query]],
):
# Keep track of how many interactions we've found for this participant
# if there is None the condition is not met
test_interactions = 0
success_by_type: Dict[QueryType, bool] = {}
for query_type, queries in participant_queries.items():
if _is_interop_test_interaction(query_type):
test_interactions += len(queries)
success_by_type[query_type] = False
for query in queries:
if 200 <= query.response.status_code < 300:
success_by_type[query_type] = True
break

self.record_note(
"test_interop_interactions",
f"Found {test_interactions} interactions with interoperability test instance for {participant_id}",
)
if test_interactions == 0:
self.record_note(
"test_interop_check_skipped",
f"Skipping check for {participant_id} because no interactions with "
f"interoperability test instance were found",
)
# If no interactions are observed, we can't determine if the test instance is available
# and the step here.
return

with self.check(
"Interoperability test instance is available", [participant_id]
) as check:
for query_type, success in success_by_type.items():
if not success:
check.record_failed(
summary=f"No successful {query_type} interaction with interoperability test instance",
severity=Severity.Medium,
details=f"Found no successful {query_type} interaction with interoperability test instance, "
f"indicating that the test instance is either not available or not properly implemented.",
)


def _is_interop_test_interaction(query_type: QueryType):
return (
query_type == QueryType.InterUSSFlightPlanningV1GetStatus
or query_type == QueryType.InterUSSFlightPlanningV1ClearArea
or query_type == QueryType.InterUSSFlightPlanningV1UpsertFlightPlan
or query_type == QueryType.InterUSSFlightPlanningV1DeleteFlightPlan
)
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ Because the modification attempt was invalid, either Flight 1 should not have be
original accepted request), or it should have been removed (because the USS rejected the replacement plan provided).



## Cleanup
### Successful flight deletion check
**[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)**
7 changes: 6 additions & 1 deletion monitoring/uss_qualifier/suites/astm/utm/f3548_21.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<th><a href="../../README.md#checked-in">Checked in</a></th>
</tr>
<tr>
<td rowspan="25" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td rowspan="26" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td><a href="../../../requirements/astm/f3548/v21.md">DSS0005</a></td>
<td>Implemented</td>
<td><a href="../../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../../scenarios/astm/utm/op_intent_access_control.md">ASTM F3548-21 UTM DSS Operational Intent Access Control</a><br><a href="../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md">Off-Nominal planning: down USS with equal priority conflicts not permitted</a><br><a href="../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
Expand All @@ -46,6 +46,11 @@
<td>Implemented</td>
<td><a href="../../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
</tr>
<tr>
<td><a href="../../../requirements/astm/f3548/v21.md">GEN0300</a></td>
<td>Implemented</td>
<td><a href="../../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
</tr>
<tr>
<td><a href="../../../requirements/astm/f3548/v21.md">GEN0310</a></td>
<td>Implemented</td>
Expand Down
7 changes: 6 additions & 1 deletion monitoring/uss_qualifier/suites/faa/uft/message_signing.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
<th><a href="../../README.md#checked-in">Checked in</a></th>
</tr>
<tr>
<td rowspan="25" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td rowspan="26" style="vertical-align:top;"><a href="../../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td><a href="../../../requirements/astm/f3548/v21.md">DSS0005</a></td>
<td>Implemented</td>
<td><a href="../../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../../scenarios/astm/utm/op_intent_access_control.md">ASTM F3548-21 UTM DSS Operational Intent Access Control</a><br><a href="../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md">Off-Nominal planning: down USS with equal priority conflicts not permitted</a><br><a href="../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
Expand All @@ -33,6 +33,11 @@
<td>Implemented</td>
<td><a href="../../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
</tr>
<tr>
<td><a href="../../../requirements/astm/f3548/v21.md">GEN0300</a></td>
<td>Implemented</td>
<td><a href="../../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
</tr>
<tr>
<td><a href="../../../requirements/astm/f3548/v21.md">GEN0310</a></td>
<td>Implemented</td>
Expand Down
7 changes: 6 additions & 1 deletion monitoring/uss_qualifier/suites/uspace/flight_auth.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<th><a href="../README.md#checked-in">Checked in</a></th>
</tr>
<tr>
<td rowspan="25" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td rowspan="26" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td><a href="../../requirements/astm/f3548/v21.md">DSS0005</a></td>
<td>Implemented</td>
<td><a href="../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../scenarios/astm/utm/op_intent_access_control.md">ASTM F3548-21 UTM DSS Operational Intent Access Control</a><br><a href="../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md">Off-Nominal planning: down USS with equal priority conflicts not permitted</a><br><a href="../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
Expand All @@ -34,6 +34,11 @@
<td>Implemented</td>
<td><a href="../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
</tr>
<tr>
<td><a href="../../requirements/astm/f3548/v21.md">GEN0300</a></td>
<td>Implemented</td>
<td><a href="../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
</tr>
<tr>
<td><a href="../../requirements/astm/f3548/v21.md">GEN0310</a></td>
<td>Implemented</td>
Expand Down
7 changes: 6 additions & 1 deletion monitoring/uss_qualifier/suites/uspace/required_services.md
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@
<td><a href="../../scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md">ASTM NetRID DSS: Concurrent Requests</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md">ASTM NetRID DSS: ISA Subscription Interactions</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_simple.md">ASTM NetRID DSS: Simple ISA</a><br><a href="../../scenarios/astm/netrid/v22a/dss/isa_validation.md">ASTM NetRID DSS: Submitted ISA Validations</a><br><a href="../../scenarios/astm/netrid/v22a/dss/subscription_simple.md">ASTM NetRID DSS: Subscription Simple</a><br><a href="../../scenarios/astm/netrid/v22a/dss/subscription_validation.md">ASTM NetRID DSS: Subscription Validation</a><br><a href="../../scenarios/astm/netrid/v22a/dss/token_validation.md">ASTM NetRID DSS: Token Validation</a></td>
</tr>
<tr>
<td rowspan="25" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td rowspan="26" style="vertical-align:top;"><a href="../../requirements/astm/f3548/v21.md">astm<br>.f3548<br>.v21</a></td>
<td><a href="../../requirements/astm/f3548/v21.md">DSS0005</a></td>
<td>Implemented</td>
<td><a href="../../scenarios/astm/utm/prep_planners.md">ASTM F3548 flight planners preparation</a><br><a href="../../scenarios/astm/utm/op_intent_access_control.md">ASTM F3548-21 UTM DSS Operational Intent Access Control</a><br><a href="../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md">Data Validation of GET operational intents by USS</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md">Nominal planning: conflict with higher priority</a><br><a href="../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md">Nominal planning: not permitted conflict with equal priority</a><br><a href="../../scenarios/astm/utm/off_nominal_planning/down_uss.md">Off-Nominal planning: down USS</a><br><a href="../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md">Off-Nominal planning: down USS with equal priority conflicts not permitted</a><br><a href="../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md">Validation of operational intents</a></td>
Expand All @@ -464,6 +464,11 @@
<td>Implemented</td>
<td><a href="../../scenarios/astm/utm/dss_interoperability.md">ASTM F3548-21 UTM DSS interoperability</a></td>
</tr>
<tr>
<td><a href="../../requirements/astm/f3548/v21.md">GEN0300</a></td>
<td>Implemented</td>
<td><a href="../../scenarios/astm/utm/aggregate_checks.md">ASTM F3548 UTM aggregate checks</a></td>
</tr>
<tr>
<td><a href="../../requirements/astm/f3548/v21.md">GEN0310</a></td>
<td>Implemented</td>
Expand Down

0 comments on commit 343ac20

Please sign in to comment.