diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index a216608076..5e852ed49c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -4,7 +4,6 @@ from monitoring.monitorlib import fetch from monitoring.monitorlib.fetch import evaluation, QueryType from monitoring.monitorlib.rid import RIDVersion -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ParticipantID from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance @@ -184,7 +183,6 @@ def _inspect_participant_queries( check.record_failed( summary=f"found {len(cleartext_queries)} cleartext http queries", details=f"unique cleartext urls: {urls}", - severity=Severity.Medium, query_timestamps=timestamps, ) @@ -220,13 +218,11 @@ def _dp_display_data_details_times_step(self): if p95 > self._rid_version.dp_details_resp_percentile95_s: check.record_failed( summary=f"95th percentile of durations for DP display_data details queries is higher than threshold", - severity=Severity.Medium, details=f"threshold: {self._rid_version.dp_details_resp_percentile95_s}s, 95th percentile: {p95:.3g}s", ) if p99 > self._rid_version.dp_details_resp_percentile99_s: check.record_failed( summary=f"99th percentile of durations for DP display_data details queries is higher than threshold", - severity=Severity.Medium, details=f"threshold: {self._rid_version.dp_details_resp_percentile99_s}s, 99th percentile: {p99:.3g}s", ) @@ -264,14 +260,12 @@ def _sp_flights_area_times_step(self): if p95 > self._rid_version.sp_data_resp_percentile95_s: check.record_failed( summary=f"95th percentile of /flights?view requests is {p95:.3g} s", - severity=Severity.Medium, details=f"expected less than {self._rid_version.sp_data_resp_percentile95_s} s, was {p95:.3g}", ) with self.check("99th percentile response time", [participant]) as check: if p99 > self._rid_version.sp_data_resp_percentile99_s: check.record_failed( summary=f"99th percentile of /flights?view requests is {p99:.3g} s", - severity=Severity.Medium, details=f"expected less than {self._rid_version.sp_data_resp_percentile99_s} s, was {p99:.3g}", ) @@ -322,13 +316,11 @@ def _dp_display_data_times_step(self): if init_95th > self._rid_version.dp_init_resp_percentile95_s: check.record_failed( summary=f"95th percentile of durations for initial DP display_data queries is higher than threshold", - severity=Severity.Medium, details=f"threshold: {self._rid_version.dp_init_resp_percentile95_s}, 95th percentile: {init_95th:.3g}", ) if init_99th > self._rid_version.dp_init_resp_percentile99_s: check.record_failed( summary=f"99th percentile of durations for initial DP display_data queries is higher than threshold", - severity=Severity.Medium, details=f"threshold: {self._rid_version.dp_init_resp_percentile99_s}, 99th percentile: {init_99th:.3g}", ) @@ -338,13 +330,11 @@ def _dp_display_data_times_step(self): if subsequent_95th > self._rid_version.dp_data_resp_percentile95_s: check.record_failed( summary=f"95th percentile of durations for subsequent DP display_data queries is higher than threshold", - severity=Severity.Medium, details=f"threshold: {self._rid_version.dp_data_resp_percentile95_s}, 95th percentile: {subsequent_95th:.3g}", ) if subsequent_99th > self._rid_version.dp_data_resp_percentile99_s: check.record_failed( summary=f"99th percentile of durations for subsequent DP display_data queries is higher than threshold", - severity=Severity.Medium, details=f"threshold: {self._rid_version.dp_data_resp_percentile99_s}, 95th percentile: {subsequent_99th:.3g}", ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index 588ae361c5..c6e5507244 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -25,7 +25,7 @@ This resource is optional. For this step, all successful display data queries made during the execution of the previous scenarios are used to compute an aggregate statistic. -#### Performance of /display_data/ requests check +#### ⚠️ Performance of /display_data/ requests check **[astm.f3411.v19.NET0460](../../../../requirements/astm/f3411/v19.md) Checks that the DP response times for the Display Application's flight details requests have a p95 and p99 that are respectively below @@ -37,12 +37,12 @@ observer and per request (identified by their URLs). For each of those, and usin `NetMinSessionLength`, the queries are split between initial and subsequent ones. The percentiles of both all the initial and all the subsequent queries are then computed to be checked. -#### Performance of /display_data initial requests check +#### ⚠️ Performance of /display_data initial requests check **[astm.f3411.v19.NET0420](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles of the durations for the initial display data queries do not exceed the respectives thresholds `NetDpInitResponse95thPercentile` and `NetDpInitResponse99thPercentile`. -#### Performance of /display_data subsequent requests check +#### ⚠️ Performance of /display_data subsequent requests check **[astm.f3411.v19.NET0440](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles of the durations for the subsequent display data queries do not exceed the respectives thresholds `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`. @@ -51,13 +51,13 @@ of the durations for the subsequent display data queries do not exceed the respe ### Performance of /flights?view requests test step -#### 95th percentile response time check +#### ⚠️ 95th percentile response time check **[astm.f3411.v19.NET0260,NetSpDataResponseTime95thPercentile](../../../../requirements/astm/f3411/v19.md)** requires that the 95th percentile of the durations for the replies to requested flights in an area does not exceed the threshold `NetSpDataResponseTime95thPercentile` (1 second). -#### 99th percentile response time check +#### ⚠️ 99th percentile response time check **[astm.f3411.v19.NET0260,NetSpDataResponseTime99thPercentile](../../../../requirements/astm/f3411/v19.md)** requires that the 99th percentile of the durations for the replies to requested flights in an area does not exceed the threshold @@ -69,7 +69,7 @@ of the durations for the replies to requested flights in an area does not exceed Inspects all record queries for their usage of https. -#### All interactions happen over https check +#### ⚠️ All interactions happen over https check If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. @@ -84,7 +84,7 @@ compliance with requirements. ### Evaluate mock USS interactions test step -#### No large Display Provider queries check +#### ⚠️ No large Display Provider queries check If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed area requested, then that participant will have violated **[astm.f3411.v19.NET0240](../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 9ed0042af1..b258be8c2c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -25,7 +25,7 @@ This resource is optional. For this step, all successful display data queries made during the execution of the previous scenarios are used to compute an aggregate statistic. -#### Performance of /display_data/ requests check +#### ⚠️ Performance of /display_data/ requests check **[astm.f3411.v22a.NET0460](../../../../requirements/astm/f3411/v22a.md)** Checks that the DP response times for the Display Application's flight details requests have a p95 and p99 that are respectively below @@ -37,12 +37,12 @@ observer and per request (identified by their URLs). For each of those, and usin `NetMinSessionLength`, the queries are split between initial and subsequent ones. The percentiles of both all the initial and all the subsequent queries are then computed to be checked. -#### Performance of /display_data initial requests check +#### ⚠️ Performance of /display_data initial requests check **[astm.f3411.v22a.NET0420](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles of the durations for the initial display data queries do not exceed the respectives thresholds `NetDpInitResponse95thPercentile` and `NetDpInitResponse99thPercentile`. -#### Performance of /display_data subsequent requests check +#### ⚠️ Performance of /display_data subsequent requests check **[astm.f3411.v22a.NET0440](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles of the durations for the subsequent display data queries do not exceed the respectives thresholds `NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`. @@ -51,13 +51,13 @@ of the durations for the subsequent display data queries do not exceed the respe ### Performance of /flights?view requests test step -#### 95th percentile response time check +#### ⚠️ 95th percentile response time check **[astm.f3411.v22a.NET0260,NetSpDataResponseTime95thPercentile](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th percentile of the durations for the replies to requested flights in an area does not exceed the threshold `NetSpDataResponseTime95thPercentile` (1 second). -#### 99th percentile response time check +#### ⚠️ 99th percentile response time check **[astm.f3411.v22a.NET0260,NetSpDataResponseTime99thPercentile](../../../../requirements/astm/f3411/v22a.md)** requires that the 99th percentile of the durations for the replies to requested flights in an area does not exceed the threshold @@ -69,7 +69,7 @@ of the durations for the replies to requested flights in an area does not exceed Inspects all record queries for their usage of https. -#### All interactions happen over https check +#### ⚠️ All interactions happen over https check If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v22a.NET0220](../../../../requirements/astm/f3411/v22a.md)** is not satisfied. @@ -84,7 +84,7 @@ compliance with requirements. ### Evaluate mock USS interactions test step -#### No large Display Provider queries check +#### ⚠️ No large Display Provider queries check If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed area requested, then that participant will have violated **[astm.f3411.v22a.NET0240](../../../../requirements/astm/f3411/v22a.md)**.