Skip to content

Commit

Permalink
[uss_qualifier/scenarios/netrid/aggregate_checks] Specify severity th…
Browse files Browse the repository at this point in the history 
…rough .md doc (interuss#756)
  • Loading branch information
@mickmis
mickmis authored Aug 20, 2024
1 parent 5cb83ab commit 0383583
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 24 deletions.
10 changes: 0 additions & 10 deletions monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from monitoring.monitorlib import fetch
from monitoring.monitorlib.fetch import evaluation, QueryType
from monitoring.monitorlib.rid import RIDVersion
from monitoring.uss_qualifier.common_data_definitions import Severity
from monitoring.uss_qualifier.configurations.configuration import ParticipantID
from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource
from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance
Expand Down Expand Up @@ -184,7 +183,6 @@ def _inspect_participant_queries(
check.record_failed(
summary=f"found {len(cleartext_queries)} cleartext http queries",
details=f"unique cleartext urls: {urls}",
severity=Severity.Medium,
query_timestamps=timestamps,
)

Expand Down Expand Up @@ -220,13 +218,11 @@ def _dp_display_data_details_times_step(self):
if p95 > self._rid_version.dp_details_resp_percentile95_s:
check.record_failed(
summary=f"95th percentile of durations for DP display_data details queries is higher than threshold",
severity=Severity.Medium,
details=f"threshold: {self._rid_version.dp_details_resp_percentile95_s}s, 95th percentile: {p95:.3g}s",
)
if p99 > self._rid_version.dp_details_resp_percentile99_s:
check.record_failed(
summary=f"99th percentile of durations for DP display_data details queries is higher than threshold",
severity=Severity.Medium,
details=f"threshold: {self._rid_version.dp_details_resp_percentile99_s}s, 99th percentile: {p99:.3g}s",
)

Expand Down Expand Up @@ -264,14 +260,12 @@ def _sp_flights_area_times_step(self):
if p95 > self._rid_version.sp_data_resp_percentile95_s:
check.record_failed(
summary=f"95th percentile of /flights?view requests is {p95:.3g} s",
severity=Severity.Medium,
details=f"expected less than {self._rid_version.sp_data_resp_percentile95_s} s, was {p95:.3g}",
)
with self.check("99th percentile response time", [participant]) as check:
if p99 > self._rid_version.sp_data_resp_percentile99_s:
check.record_failed(
summary=f"99th percentile of /flights?view requests is {p99:.3g} s",
severity=Severity.Medium,
details=f"expected less than {self._rid_version.sp_data_resp_percentile99_s} s, was {p99:.3g}",
)

Expand Down Expand Up @@ -322,13 +316,11 @@ def _dp_display_data_times_step(self):
if init_95th > self._rid_version.dp_init_resp_percentile95_s:
check.record_failed(
summary=f"95th percentile of durations for initial DP display_data queries is higher than threshold",
severity=Severity.Medium,
details=f"threshold: {self._rid_version.dp_init_resp_percentile95_s}, 95th percentile: {init_95th:.3g}",
)
if init_99th > self._rid_version.dp_init_resp_percentile99_s:
check.record_failed(
summary=f"99th percentile of durations for initial DP display_data queries is higher than threshold",
severity=Severity.Medium,
details=f"threshold: {self._rid_version.dp_init_resp_percentile99_s}, 99th percentile: {init_99th:.3g}",
)

Expand All @@ -338,13 +330,11 @@ def _dp_display_data_times_step(self):
if subsequent_95th > self._rid_version.dp_data_resp_percentile95_s:
check.record_failed(
summary=f"95th percentile of durations for subsequent DP display_data queries is higher than threshold",
severity=Severity.Medium,
details=f"threshold: {self._rid_version.dp_data_resp_percentile95_s}, 95th percentile: {subsequent_95th:.3g}",
)
if subsequent_99th > self._rid_version.dp_data_resp_percentile99_s:
check.record_failed(
summary=f"99th percentile of durations for subsequent DP display_data queries is higher than threshold",
severity=Severity.Medium,
details=f"threshold: {self._rid_version.dp_data_resp_percentile99_s}, 95th percentile: {subsequent_99th:.3g}",
)

Expand Down
14 changes: 7 additions & 7 deletions monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ This resource is optional.

For this step, all successful display data queries made during the execution of the previous scenarios are used to compute an aggregate statistic.

#### Performance of /display_data/<flight_id> requests check
#### ⚠️ Performance of /display_data/<flight_id> requests check

**[astm.f3411.v19.NET0460](../../../../requirements/astm/f3411/v19.md) Checks that the DP response times for the
Display Application's flight details requests have a p95 and p99 that are respectively below
Expand All @@ -37,12 +37,12 @@ observer and per request (identified by their URLs). For each of those, and usin
`NetMinSessionLength`, the queries are split between initial and subsequent ones.
The percentiles of both all the initial and all the subsequent queries are then computed to be checked.

#### Performance of /display_data initial requests check
#### ⚠️ Performance of /display_data initial requests check
**[astm.f3411.v19.NET0420](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles
of the durations for the initial display data queries do not exceed the respectives thresholds
`NetDpInitResponse95thPercentile` and `NetDpInitResponse99thPercentile`.

#### Performance of /display_data subsequent requests check
#### ⚠️ Performance of /display_data subsequent requests check
**[astm.f3411.v19.NET0440](../../../../requirements/astm/f3411/v19.md)** requires that the 95th and 99th percentiles
of the durations for the subsequent display data queries do not exceed the respectives thresholds
`NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`.
Expand All @@ -51,13 +51,13 @@ of the durations for the subsequent display data queries do not exceed the respe

### Performance of /flights?view requests test step

#### 95th percentile response time check
#### ⚠️ 95th percentile response time check

**[astm.f3411.v19.NET0260,NetSpDataResponseTime95thPercentile](../../../../requirements/astm/f3411/v19.md)** requires that the 95th percentile
of the durations for the replies to requested flights in an area does not exceed the threshold
`NetSpDataResponseTime95thPercentile` (1 second).

#### 99th percentile response time check
#### ⚠️ 99th percentile response time check

**[astm.f3411.v19.NET0260,NetSpDataResponseTime99thPercentile](../../../../requirements/astm/f3411/v19.md)** requires that the 99th percentile
of the durations for the replies to requested flights in an area does not exceed the threshold
Expand All @@ -69,7 +69,7 @@ of the durations for the replies to requested flights in an area does not exceed

Inspects all record queries for their usage of https.

#### All interactions happen over https check
#### ⚠️ All interactions happen over https check

If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied.

Expand All @@ -84,7 +84,7 @@ compliance with requirements.

### Evaluate mock USS interactions test step

#### No large Display Provider queries check
#### ⚠️ No large Display Provider queries check

If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed
area requested, then that participant will have violated **[astm.f3411.v19.NET0240](../../../../requirements/astm/f3411/v19.md)**.
Expand Down
14 changes: 7 additions & 7 deletions monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ This resource is optional.

For this step, all successful display data queries made during the execution of the previous scenarios are used to compute an aggregate statistic.

#### Performance of /display_data/<flight_id> requests check
#### ⚠️ Performance of /display_data/<flight_id> requests check

**[astm.f3411.v22a.NET0460](../../../../requirements/astm/f3411/v22a.md)** Checks that the DP response times for the
Display Application's flight details requests have a p95 and p99 that are respectively below
Expand All @@ -37,12 +37,12 @@ observer and per request (identified by their URLs). For each of those, and usin
`NetMinSessionLength`, the queries are split between initial and subsequent ones.
The percentiles of both all the initial and all the subsequent queries are then computed to be checked.

#### Performance of /display_data initial requests check
#### ⚠️ Performance of /display_data initial requests check
**[astm.f3411.v22a.NET0420](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles
of the durations for the initial display data queries do not exceed the respectives thresholds
`NetDpInitResponse95thPercentile` and `NetDpInitResponse99thPercentile`.

#### Performance of /display_data subsequent requests check
#### ⚠️ Performance of /display_data subsequent requests check
**[astm.f3411.v22a.NET0440](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th and 99th percentiles
of the durations for the subsequent display data queries do not exceed the respectives thresholds
`NetDpDataResponse95thPercentile` and `NetDpDataResponse99thPercentile`.
Expand All @@ -51,13 +51,13 @@ of the durations for the subsequent display data queries do not exceed the respe

### Performance of /flights?view requests test step

#### 95th percentile response time check
#### ⚠️ 95th percentile response time check

**[astm.f3411.v22a.NET0260,NetSpDataResponseTime95thPercentile](../../../../requirements/astm/f3411/v22a.md)** requires that the 95th percentile
of the durations for the replies to requested flights in an area does not exceed the threshold
`NetSpDataResponseTime95thPercentile` (1 second).

#### 99th percentile response time check
#### ⚠️ 99th percentile response time check

**[astm.f3411.v22a.NET0260,NetSpDataResponseTime99thPercentile](../../../../requirements/astm/f3411/v22a.md)** requires that the 99th percentile
of the durations for the replies to requested flights in an area does not exceed the threshold
Expand All @@ -69,7 +69,7 @@ of the durations for the replies to requested flights in an area does not exceed

Inspects all record queries for their usage of https.

#### All interactions happen over https check
#### ⚠️ All interactions happen over https check

If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v22a.NET0220](../../../../requirements/astm/f3411/v22a.md)** is not satisfied.

Expand All @@ -84,7 +84,7 @@ compliance with requirements.

### Evaluate mock USS interactions test step

#### No large Display Provider queries check
#### ⚠️ No large Display Provider queries check

If one of the Display Provider test participants was found to have sent a query to mock_uss with a larger-than-allowed
area requested, then that participant will have violated **[astm.f3411.v22a.NET0240](../../../../requirements/astm/f3411/v22a.md)**.
Expand Down

0 comments on commit 0383583

Please sign in to comment.