diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 28a5615c50..3c29d18e02 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -269,12 +269,15 @@ def describe_query( query_type: Optional[QueryType] = None, server_id: Optional[str] = None, ) -> Query: - return Query( + query = Query( request=describe_request(resp.request, initiated_at), response=describe_response(resp), - server_id=server_id, - query_type=query_type, ) + if query_type is not None: + query.query_type = query_type + if server_id is not None: + query.server_id = server_id + return query def query_and_describe( @@ -282,6 +285,7 @@ def query_and_describe( verb: str, url: str, query_type: Optional[QueryType] = None, + server_id: Optional[str] = None, **kwargs, ) -> Query: """Attempt to perform a query, and then describe the results of that attempt. @@ -294,6 +298,7 @@ def query_and_describe( verb: HTTP verb to perform at the specified URL. url: URL to query. query_type: If specified, the known type of query that this is. + server_id: If specified, the participant identifier of the server being queried. **kwargs: Any keyword arguments that should be applied to the .request method when invoking it. Returns: @@ -305,7 +310,6 @@ def query_and_describe( else: utm_session = True req_kwargs = kwargs.copy() - req_kwargs.pop("server_id", None) if "timeout" not in req_kwargs: req_kwargs["timeout"] = TIMEOUTS @@ -320,7 +324,7 @@ def query_and_describe( client.request(verb, url, **req_kwargs), t0, query_type=query_type, - server_id=kwargs.get("server_id", None), + server_id=server_id, ) except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}" @@ -350,7 +354,7 @@ def query_and_describe( elapsed_s=(t1 - t0).total_seconds(), reported=StringBasedDateTime(t1), ), - server_id=kwargs.get("server_id", None), + server_id=server_id, ) if query_type is not None: result.query_type = query_type diff --git a/monitoring/uss_qualifier/resources/astm/f3411/dss.py b/monitoring/uss_qualifier/resources/astm/f3411/dss.py index a468df5693..4b77399c81 100644 --- a/monitoring/uss_qualifier/resources/astm/f3411/dss.py +++ b/monitoring/uss_qualifier/resources/astm/f3411/dss.py @@ -69,6 +69,7 @@ def __init__( if has_private_address is not None: self.has_private_address = has_private_address + self.local_debug = True if local_debug is not None: self.local_debug = local_debug diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index a800e9f019..ade57069a2 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -22,6 +22,8 @@ NetRIDServiceProvider, ) +from loguru import logger + class AggregateChecks(ReportEvaluationScenario): _rid_version: RIDVersion @@ -73,6 +75,7 @@ def __init__( participant: list() for participant in self._participants_by_base_url.values() } + for query in self._queries: for base_url, participant in self._participants_by_base_url.items(): if query.request.url.startswith(base_url): @@ -141,7 +144,6 @@ def run(self): self.end_test_scenario() def _verify_https_everywhere(self): - for participant_id, participant_queries in self._queries_by_participant.items(): self._inspect_participant_queries(participant_id, participant_queries) @@ -155,12 +157,9 @@ def _verify_https_everywhere(self): # TODO clean this up: this is an internal requirement and not a check, # leaving as-is during development to make sure the test-suite runs but we know about unattributed queries # ultimately this check could go into the constructor and blow things up early - self.record_note( - "unattributed_queries", - f"found {len(unattr_queries)} unattributed queries", - ) + with self.check( - "All interactions happen over https", + "No unattributed queries", [], ) as check: check.record_failed( @@ -170,13 +169,12 @@ def _verify_https_everywhere(self): def _inspect_participant_queries( self, participant_id: str, participant_queries: List[fetch.Query] ): - found_cleartext_query = False + cleartext_queries = [] for query in participant_queries: if query.request.url.startswith("http://"): - found_cleartext_query = True if participant_id not in self._debug_mode_usses: - self.record_note( - "cleartext-query", + cleartext_queries.append(query) + logger.info( f"query is not https: {participant_id} - {query.request.url}", ) @@ -185,9 +183,17 @@ def _inspect_participant_queries( "All interactions happen over https", [participant_id], ) as check: - if found_cleartext_query: + if len(cleartext_queries) > 0: + timestamps = [ + q.request.initiated_at.datetime for q in cleartext_queries + ] + urls = set([q.request.url for q in cleartext_queries]) check.record_failed( - "found at least one cleartext http query", Severity.Medium + summary=f"found {len(cleartext_queries)} cleartext http queries", + details=f"unique cleartext urls: {urls}", + severity=Severity.Medium, + participants=[participant_id], + query_timestamps=timestamps, ) else: self.record_note( @@ -203,7 +209,6 @@ def _dp_display_data_details_times_step(self): for participant, all_queries in self._queries_by_participant.items(): relevant_queries: List[fetch.Query] = list() for query in all_queries: - if ( query.status_code == 200 and query.has_field_with_value("query_type") @@ -303,7 +308,6 @@ def _dp_display_data_times_step(self): pattern = re.compile(r"/display_data\?view=(-?\d+(.\d+)?,){3}-?\d+(.\d+)?") for participant, all_queries in self._queries_by_participant.items(): - # identify successful display_data queries relevant_queries: List[fetch.Query] = list() for query in all_queries: diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index fe75b44ebf..ffbd2d878e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -255,6 +255,7 @@ def _evaluate_and_test_authentication( get_details=True, rid_version=self._rid_version, session=self._dss.client, + server_id=self._dss.participant_id, ) # We fish out the queries that were used to grab the flights from the SP, # and attempt to re-query without credentials. This should fail. @@ -285,6 +286,7 @@ def _evaluate_and_test_authentication( url=fq.query.request.url, json=fq.query.request.json, data=fq.query.request.body, + server_id=self._dss.participant_id, ) logger.info( f"Repeating query to {fq.query.request.url} without credentials" diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md index f729ce4938..fb7e485c56 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md @@ -76,9 +76,16 @@ TODO: Implement this check ### Verify https is in use test step -Inspects all record queries for their usage of https. If resources such as a service provide, observer or DSS are marked -as being in "local debug" mode, they may serve requests over https without breaking the test suite. +Inspects all record queries for their usage of https. If services such as a service provider, observer or DSS are marked +as being in "local debug" mode, they may serve requests over http without producing failed checks despite their lack of encryption. #### All interactions happen over https check If non-encrypted interactions such as plaintext queries over http are allowed, **[astm.f3411.v19.NET0220](../../../../requirements/astm/f3411/v19.md)** is not satisfied. + +#### No unattributed queries check + +All queries must have been attributed to a participant: an unattributed query means that one of the test cases has not +properly recorded to which participant it was made. + +This is an internal requirement and does not necessarily imply that there is a problem with the participants under test. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md index 21dbb894e7..2e328f175b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md @@ -76,8 +76,15 @@ TODO: Implement this check ### Verify https is in use test step -Inspects all record queries for their usage of https. If resources such as a service provide, observer or DSS are marked -as being in "local debug" mode, they may serve requests over https without breaking the test suite. +Inspects all record queries for their usage of https. If services such as a service provider, observer or DSS are marked +as being in "local debug" mode, they may serve requests over http without producing failed checks despite their lack of encryption. + +#### No unattributed queries check + +All queries must have been attributed to a participant: an unattributed query means that one of the test cases has not +properly recorded to which participant it was made. + +This is an internal requirement and does not necessarily imply that there is a problem with the participants under test. #### All interactions happen over https check diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 089ea93df4..b8795203e4 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -21,7 +21,7 @@ Checked in - astm
.f3411
.v19 + astm
.f3411
.v19 A2-6-1,1a Implemented ASTM F3411-19 NetRID DSS interoperability @@ -166,6 +166,11 @@ Implemented ASTM F3411-19 NetRID DSS interoperability + + NET0220 + Implemented + ASTM F3411-19 NetRID aggregate checks + NET0240 Planned diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 6096cf8563..96bddbbf33 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -20,6 +20,12 @@ Status Checked in + + astm
.f3411
.v19 + NET0220 + Implemented + ASTM F3411-22a NetRID aggregate checks + astm
.f3411
.v22a A2-6-1,1a diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index ec7130edb8..45065fac1f 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -15,6 +15,12 @@ Status Checked in + + astm
.f3411
.v19 + NET0220 + Implemented + ASTM F3411-22a NetRID aggregate checks + astm
.f3411
.v22a A2-6-1,1a diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 1e24a9d65f..54f85ab0c0 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -16,6 +16,12 @@ Status Checked in + + astm
.f3411
.v19 + NET0220 + Implemented + ASTM F3411-22a NetRID aggregate checks + astm
.f3411
.v22a A2-6-1,1a