From c8a1a792495c30d8b01c9bca4e48dd028293947f Mon Sep 17 00:00:00 2001 From: Punam Verma Date: Thu, 7 Sep 2023 14:15:21 -0700 Subject: [PATCH] [uss_qualifier/mock_uss] Add interaction logging functionality (#186) * Adding interuss logging in mock_uss * delete interuss logs * Removing unwanted var * Fixing lint issues * Fixing lint issues * fix return value * fix lint issue * Fixing the PR comments about renaming variables and files * Renaming variable as per PR comment * As per PR comments - 1. Switched to hook pattern for logging interactions 2. Removed default value for log dir, and response status code if dir not found 3. Fixed num of files deleted log * As per PR comment - using json format to write interaction logs * Fix as per PR comments - 1. Added a container for interaction_logging in docker compose 2. return 500 on Errors in GET /logs 3. interaction_logging specific config moved to its own package * Suggest updates * Add filter for incoming interaction logging * Fixing an error found by CI --------- Co-authored-by: Benjamin Pelletier --- monitoring/mock_uss/README.md | 1 + monitoring/mock_uss/__init__.py | 7 +- monitoring/mock_uss/docker-compose.yaml | 35 +++++- .../mock_uss/interaction_logging/__init__.py | 1 + .../mock_uss/interaction_logging/config.py | 11 ++ .../interaction_logging/interactions.py | 28 +++++ .../mock_uss/interaction_logging/logger.py | 72 ++++++++++++ .../routes_interactions_log.py | 84 ++++++++++++++ monitoring/mock_uss/run_locally.sh | 21 +++- monitoring/mock_uss/scdsc/routes_injection.py | 7 +- monitoring/monitorlib/clients/__init__.py | 22 ++++ monitoring/monitorlib/clients/scd.py | 30 +++-- monitoring/monitorlib/fetch/__init__.py | 108 +++++++++++++++++- .../monitoring/monitorlib/fetch/Query.json | 60 ++++++++-- 14 files changed, 454 insertions(+), 33 deletions(-) create mode 100644 monitoring/mock_uss/interaction_logging/__init__.py create mode 100644 monitoring/mock_uss/interaction_logging/config.py create mode 100644 monitoring/mock_uss/interaction_logging/interactions.py create mode 100644 monitoring/mock_uss/interaction_logging/logger.py create mode 100644 monitoring/mock_uss/interaction_logging/routes_interactions_log.py diff --git a/monitoring/mock_uss/README.md b/monitoring/mock_uss/README.md index 289e7a2528..d59e0fac81 100644 --- a/monitoring/mock_uss/README.md +++ b/monitoring/mock_uss/README.md @@ -21,6 +21,7 @@ The available functionality sets are: * [`ridsp`](ridsp): Remote ID Service Provider * [`scdsc`](scdsc): ASTM F3548 strategic coordinator * [`tracer`](tracer): Interoperability ecosystem tracer logger +* [`interaction_logging`](interaction_logging): Enables logging of the [interuss](https://github.com/astm-utm/Protocol/blob/master/utm.yaml) interactions between mock_uss and other uss participants ## Local deployment diff --git a/monitoring/mock_uss/__init__.py b/monitoring/mock_uss/__init__.py index 680284d6a4..94587f3b96 100644 --- a/monitoring/mock_uss/__init__.py +++ b/monitoring/mock_uss/__init__.py @@ -1,7 +1,6 @@ import inspect import os from typing import Any, Optional, Callable - from loguru import logger from monitoring.mock_uss.server import MockUSS @@ -13,6 +12,7 @@ SERVICE_MESSAGESIGNING = "msgsigning" SERVICE_ATPROXY_CLIENT = "atproxy_client" SERVICE_TRACER = "tracer" +SERVICE_INTERACTION_LOGGING = "interaction_logging" webapp = MockUSS(__name__) enabled_services = set() @@ -88,6 +88,11 @@ def require_config_value(config_key: str) -> None: from monitoring.mock_uss import msgsigning from monitoring.mock_uss.msgsigning import routes as msgsigning_routes +if SERVICE_INTERACTION_LOGGING in webapp.config[config.KEY_SERVICES]: + enabled_services.add(SERVICE_INTERACTION_LOGGING) + from monitoring.mock_uss.interaction_logging import logger as interactions_logger + from monitoring.mock_uss.interaction_logging import routes_interactions_log + if SERVICE_ATPROXY_CLIENT in webapp.config[config.KEY_SERVICES]: enabled_services.add(SERVICE_ATPROXY_CLIENT) from monitoring.mock_uss import atproxy_client diff --git a/monitoring/mock_uss/docker-compose.yaml b/monitoring/mock_uss/docker-compose.yaml index 6f5f284f38..38d0601430 100644 --- a/monitoring/mock_uss/docker-compose.yaml +++ b/monitoring/mock_uss/docker-compose.yaml @@ -18,7 +18,9 @@ services: - MOCK_USS_PUBLIC_KEY=/var/test-certs/auth2.pem - MOCK_USS_TOKEN_AUDIENCE=scdsc.uss1.localutm,localhost,host.docker.internal - MOCK_USS_BASE_URL=http://scdsc.uss1.localutm - - MOCK_USS_SERVICES=scdsc + # TODO: remove interaction_logging once dedicated mock_uss is involved in tests + - MOCK_USS_SERVICES=scdsc,interaction_logging + - MOCK_USS_INTERACTIONS_LOG_DIR=output/scdsc_a_interaction_logs - MOCK_USS_PORT=80 expose: - 80 @@ -26,6 +28,8 @@ services: - 8074:80 volumes: - ../../build/test-certs:/var/test-certs:ro + - ./output/scdsc_a_interaction_logs:/app/monitoring/mock_uss/output/scdsc_a_interaction_logs + user: "${UID_GID}" restart: always networks: - interop_ecosystem_network @@ -212,6 +216,35 @@ services: extra_hosts: - host.docker.internal:host-gateway + mock_uss_scdsc_interaction_log: + container_name: mock_uss_scdsc_interaction_log + hostname: scdsc.log.uss6.localutm + image: interuss/monitoring + command: mock_uss/start.sh + environment: + - MOCK_USS_AUTH_SPEC=DummyOAuth(http://oauth.authority.localutm:8085/token,uss6) + - MOCK_USS_DSS_URL=http://dss.uss1.localutm + - MOCK_USS_PUBLIC_KEY=/var/test-certs/auth2.pem + - MOCK_USS_TOKEN_AUDIENCE=scdsc.log.uss6.localutm,localhost,host.docker.internal + - MOCK_USS_BASE_URL=http://scdsc.log.uss6.localutm + - MOCK_USS_SERVICES=scdsc,interaction_logging + - MOCK_USS_INTERACTIONS_LOG_DIR=output/scdsc_interaction_logs + - MOCK_USS_PORT=80 + expose: + - 80 + ports: + - 8095:80 + volumes: + - ../../build/test-certs:/var/test-certs:ro + - ./output/scdsc_interaction_logs:/app/monitoring/mock_uss/output/scdsc_interaction_logs + user: "${UID_GID}" + restart: always + networks: + - interop_ecosystem_network + extra_hosts: + - host.docker.internal:host-gateway + + # atproxy is currently disabled to troubleshoot occasional connection issues # atproxy: # container_name: atproxy diff --git a/monitoring/mock_uss/interaction_logging/__init__.py b/monitoring/mock_uss/interaction_logging/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/monitoring/mock_uss/interaction_logging/__init__.py @@ -0,0 +1 @@ + diff --git a/monitoring/mock_uss/interaction_logging/config.py b/monitoring/mock_uss/interaction_logging/config.py new file mode 100644 index 0000000000..1fabaedb31 --- /dev/null +++ b/monitoring/mock_uss/interaction_logging/config.py @@ -0,0 +1,11 @@ +import os.path + +from monitoring.mock_uss import import_environment_variable, webapp + +KEY_INTERACTIONS_LOG_DIR = "MOCK_USS_INTERACTIONS_LOG_DIR" + +import_environment_variable(KEY_INTERACTIONS_LOG_DIR) + +_full_path = os.path.abspath(webapp.config[KEY_INTERACTIONS_LOG_DIR]) +if not os.path.exists(_full_path): + raise ValueError(f"MOCK_USS_INTERACTIONS_LOG_DIR {_full_path} does not exist") diff --git a/monitoring/mock_uss/interaction_logging/interactions.py b/monitoring/mock_uss/interaction_logging/interactions.py new file mode 100644 index 0000000000..fdfc62fb57 --- /dev/null +++ b/monitoring/mock_uss/interaction_logging/interactions.py @@ -0,0 +1,28 @@ +from enum import Enum +from typing import List + +from implicitdict import ImplicitDict +import yaml +from yaml.representer import Representer + +from monitoring.monitorlib.fetch import Query + + +class QueryDirection(str, Enum): + Incoming = "Incoming" + """The query originated from a remote client and was handled by the system reporting the interaction.""" + + Outgoing = "Outgoing" + """The system reporting the interaction initiated the query to a remote server.""" + + +class Interaction(ImplicitDict): + query: Query + direction: QueryDirection + + +yaml.add_representer(Interaction, Representer.represent_dict) + + +class ListLogsResponse(ImplicitDict): + interactions: List[Interaction] diff --git a/monitoring/mock_uss/interaction_logging/logger.py b/monitoring/mock_uss/interaction_logging/logger.py new file mode 100644 index 0000000000..e09acf5b3f --- /dev/null +++ b/monitoring/mock_uss/interaction_logging/logger.py @@ -0,0 +1,72 @@ +import os +import datetime + +import flask +import json +from loguru import logger + +from monitoring.mock_uss import webapp, require_config_value +from monitoring.mock_uss.interaction_logging.config import KEY_INTERACTIONS_LOG_DIR +from monitoring.mock_uss.interaction_logging.interactions import ( + Interaction, + QueryDirection, +) +from monitoring.monitorlib.clients import QueryHook, query_hooks +from monitoring.monitorlib.fetch import Query, describe_flask_query, QueryType + +require_config_value(KEY_INTERACTIONS_LOG_DIR) + + +def log_interaction(direction: QueryDirection, query: Query) -> None: + """Logs the REST calls between Mock USS to SUT + Args: + direction: Whether this interaction was initiated or handled by this system. + query: Full description of the interaction to log. + """ + interaction: Interaction = Interaction(query=query, direction=direction) + method = query.request.method + log_file(f"{direction}_{method}", interaction) + + +def log_file(code: str, content: Interaction) -> None: + log_path = webapp.config[KEY_INTERACTIONS_LOG_DIR] + n = len(os.listdir(log_path)) + basename = "{:06d}_{}_{}".format( + n, code, datetime.datetime.now().strftime("%H%M%S_%f") + ) + logname = "{}.json".format(basename) + fullname = os.path.join(log_path, logname) + + with open(fullname, "w") as f: + json.dump(content, f) + + +class InteractionLoggingHook(QueryHook): + def on_query(self, query: Query) -> None: + # TODO: Make this configurable instead of hardcoding exactly these query types + if "query_type" in query and query.query_type in { + QueryType.F3548v21USSGetOperationalIntentDetails, + QueryType.F3548v21USSNotifyOperationalIntentDetailsChanged, + }: + log_interaction(QueryDirection.Outgoing, query) + + +query_hooks.append(InteractionLoggingHook()) + + +# https://stackoverflow.com/a/67856316 +@webapp.before_request +def interaction_log_before_request(): + flask.Flask.custom_profiler = {"start": datetime.datetime.utcnow()} + + +@webapp.after_request +def interaction_log_after_request(response): + elapsed_s = ( + datetime.datetime.utcnow() - flask.current_app.custom_profiler["start"] + ).total_seconds() + # TODO: Make this configurable instead of hardcoding exactly these query types + if "/uss/v1/" in flask.request.url_rule.rule: + query = describe_flask_query(flask.request, response, elapsed_s) + log_interaction(QueryDirection.Incoming, query) + return response diff --git a/monitoring/mock_uss/interaction_logging/routes_interactions_log.py b/monitoring/mock_uss/interaction_logging/routes_interactions_log.py new file mode 100644 index 0000000000..4acdb573ed --- /dev/null +++ b/monitoring/mock_uss/interaction_logging/routes_interactions_log.py @@ -0,0 +1,84 @@ +from typing import Tuple, List +import json +import os +from implicitdict import ImplicitDict, StringBasedDateTime +from flask import request, jsonify +from loguru import logger + +from monitoring.mock_uss import webapp +from monitoring.mock_uss.auth import requires_scope +from monitoring.monitorlib.scd_automated_testing.scd_injection_api import ( + SCOPE_SCD_QUALIFIER_INJECT, +) +from monitoring.mock_uss.interaction_logging.interactions import ( + Interaction, + ListLogsResponse, +) + +from monitoring.mock_uss.interaction_logging.config import KEY_INTERACTIONS_LOG_DIR + + +@webapp.route("/mock_uss/interuss_logging/logs", methods=["GET"]) +@requires_scope([SCOPE_SCD_QUALIFIER_INJECT]) +def interaction_logs() -> Tuple[str, int]: + """ + Returns all the interaction logs with requests that were + received or initiated between 'from_time' and now + Eg - http:/.../mock_uss/interuss/log?from_time=2023-08-30T20:48:21.900000Z + """ + from_time_param = request.args.get("from_time", "1900-01-01T00:00:00Z") + from_time = StringBasedDateTime(from_time_param) + log_path = webapp.config[KEY_INTERACTIONS_LOG_DIR] + + if not os.path.exists(log_path): + raise ValueError(f"Configured log path {log_path} does not exist") + + interactions: List[Interaction] = [] + for fname in os.listdir(log_path): + with open(os.path.join(log_path, fname), "r") as f: + try: + obj = json.load(f) + interaction = ImplicitDict.parse(obj, Interaction) + if ( + ("received_at" in interaction.query.request) + and interaction.query.request.received_at.datetime + >= from_time.datetime + ): + interactions.append(interaction) + elif ( + "initiated_at" in interaction.query.request + and interaction.query.request.initiated_at.datetime + >= from_time.datetime + ): + interactions.append(interaction) + else: + raise ValueError( + f"There is no received_at or initiated_at field in the request in {fname}" + ) + + except (KeyError, ValueError) as e: + msg = f"Error occurred in reading interaction from file {fname}: {e}" + raise type(e)(msg) + + return jsonify(ListLogsResponse(interactions=interactions)), 200 + + +@webapp.route("/mock_uss/interuss_logging/logs", methods=["DELETE"]) +@requires_scope([SCOPE_SCD_QUALIFIER_INJECT]) +def delete_interaction_logs() -> Tuple[str, int]: + """Deletes all the files under the logging directory""" + log_path = webapp.config[KEY_INTERACTIONS_LOG_DIR] + + if not os.path.exists(log_path): + raise ValueError(f"Configured log path {log_path} does not exist") + + logger.debug(f"Number of files in {log_path}: {len(os.listdir(log_path))}") + + num_removed = 0 + for file in os.listdir(log_path): + file_path = os.path.join(log_path, file) + os.remove(file_path) + logger.debug(f"Removed log file - {file_path}") + num_removed = num_removed + 1 + + return f"Removed {num_removed} files", 200 diff --git a/monitoring/mock_uss/run_locally.sh b/monitoring/mock_uss/run_locally.sh index 96968cb358..27c1abdbc3 100755 --- a/monitoring/mock_uss/run_locally.sh +++ b/monitoring/mock_uss/run_locally.sh @@ -31,15 +31,24 @@ elif [[ "$DC_COMMAND" == "debug" ]]; then export DEBUG_ON=1 fi -mkdir -p output/tracer UID_GID="$(id -u):$(id -g)" export UID_GID echo "DC_COMMAND is ${DC_COMMAND}" -if [[ "$DC_COMMAND" == up* ]]; then - echo "Cleaning up past tracer logs" - # Prevent logs from building up too much by default - find "output/tracer" -name "*.yaml" -exec rm {} \; -fi + +declare -a log_folders=( \ + "output/tracer" \ + "output/scdsc_a_interaction_logs" \ + "output/scdsc_interaction_logs" \ + ) +for log_folder in "${log_folders[@]}"; do + mkdir -p "$log_folder" + if [[ "$DC_COMMAND" == up* ]]; then + echo "Cleaning up past logs in $log_folder" + # Prevent logs from building up too much by default + find "$log_folder" -name "*.yaml" -exec rm {} \; + find "$log_folder" -name "*.json" -exec rm {} \; + fi +done # shellcheck disable=SC2086 docker compose -f docker-compose.yaml -p mocks $DC_COMMAND $DC_OPTIONS diff --git a/monitoring/mock_uss/scdsc/routes_injection.py b/monitoring/mock_uss/scdsc/routes_injection.py index f61e03fe83..0254d1c57b 100644 --- a/monitoring/mock_uss/scdsc/routes_injection.py +++ b/monitoring/mock_uss/scdsc/routes_injection.py @@ -85,11 +85,10 @@ def query_operational_intents( updated_op_intents = [] for op_intent_ref in get_details_for: - updated_op_intents.append( - scd_client.get_operational_intent_details( - utm_client, op_intent_ref.uss_base_url, op_intent_ref.id - ) + op_intent, _ = scd_client.get_operational_intent_details( + utm_client, op_intent_ref.uss_base_url, op_intent_ref.id ) + updated_op_intents.append(op_intent) result.extend(updated_op_intents) with db as tx: diff --git a/monitoring/monitorlib/clients/__init__.py b/monitoring/monitorlib/clients/__init__.py index e69de29bb2..7a00b334bf 100644 --- a/monitoring/monitorlib/clients/__init__.py +++ b/monitoring/monitorlib/clients/__init__.py @@ -0,0 +1,22 @@ +from abc import ABC +from typing import List + +from monitoring.monitorlib.fetch import Query + + +class QueryHook(ABC): + def on_query(self, query: Query) -> None: + """Called whenever a client performs a query and this hook is included in query_hooks. + + Args: + query: Query that was performed. + """ + raise NotImplementedError("QueryHook subclass did not implement on_query") + + +query_hooks: List[QueryHook] = [] + + +def call_query_hooks(query: Query) -> None: + for hook in query_hooks: + hook.on_query(query=query) diff --git a/monitoring/monitorlib/clients/scd.py b/monitoring/monitorlib/clients/scd.py index 58d216ebe6..89f27e16e4 100644 --- a/monitoring/monitorlib/clients/scd.py +++ b/monitoring/monitorlib/clients/scd.py @@ -1,9 +1,11 @@ -from typing import List, Optional +from typing import List, Optional, Tuple from monitoring.monitorlib import fetch, scd -from monitoring.monitorlib.fetch import QueryError +from monitoring.monitorlib.clients import call_query_hooks +from monitoring.monitorlib.fetch import QueryError, Query, QueryType from monitoring.monitorlib.infrastructure import UTMClientSession from implicitdict import ImplicitDict +from loguru import logger # === DSS operations defined in ASTM API === @@ -137,10 +139,17 @@ def delete_operational_intent_reference( def get_operational_intent_details( utm_client: UTMClientSession, uss_base_url: str, id: str -) -> scd.OperationalIntent: +) -> Tuple[scd.OperationalIntent, Query]: url = f"{uss_base_url}/uss/v1/operational_intents/{id}" subject = f"getOperationalIntentDetails from {url}" - query = fetch.query_and_describe(utm_client, "GET", url, scope=scd.SCOPE_SC) + query = fetch.query_and_describe( + utm_client, + "GET", + url, + QueryType.F3548v21USSGetOperationalIntentDetails, + scope=scd.SCOPE_SC, + ) + call_query_hooks(query) if query.status_code != 200: raise QueryError( msg="{} failed {}:\n{}".format( @@ -160,19 +169,25 @@ def get_operational_intent_details( raise QueryError( msg=f"{subject} response contained invalid JSON: {str(e)}", queries=[query] ) - return resp_body.operational_intent + return resp_body.operational_intent, query def notify_operational_intent_details_changed( utm_client: UTMClientSession, uss_base_url: str, update: scd.PutOperationalIntentDetailsParameters, -) -> None: +) -> Query: url = f"{uss_base_url}/uss/v1/operational_intents" subject = f"notifyOperationalIntentDetailsChanged to {url}" query = fetch.query_and_describe( - utm_client, "POST", url, json=update, scope=scd.SCOPE_SC + utm_client, + "POST", + url, + QueryType.F3548v21USSNotifyOperationalIntentDetailsChanged, + json=update, + scope=scd.SCOPE_SC, ) + call_query_hooks(query) if query.status_code != 204 and query.status_code != 200: raise QueryError( msg="{} failed {}:\n{}".format( @@ -180,6 +195,7 @@ def notify_operational_intent_details_changed( ), queries=[query], ) + return query # === Custom actions === diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py index 17c5c73f55..973a7dd0a3 100644 --- a/monitoring/monitorlib/fetch/__init__.py +++ b/monitoring/monitorlib/fetch/__init__.py @@ -139,12 +139,79 @@ def describe_response(resp: requests.Response) -> ResponseDescription: return ResponseDescription(**kwargs) +def describe_flask_response(resp: flask.Response, elapsed_s: float): + headers = {k: v for k, v in resp.headers.items()} + kwargs = { + "code": resp.status_code, + "headers": headers, + "reported": StringBasedDateTime(datetime.datetime.utcnow()), + "elapsed_s": elapsed_s, + } + try: + kwargs["json"] = resp.get_json() + except ValueError: + kwargs["body"] = resp.get_data(as_text=True) + return ResponseDescription(**kwargs) + + class QueryType(str, Enum): F3411v22aFlights = "astm.f3411.v22a.sp.flights" F3411v19Flights = "astm.f3411.v19.sp.flights" F3411v22aFlightDetails = "astm.f3411.v22a.sp.flight_details" F3411v19aFlightDetails = "astm.f3411.v19.sp.flight_details" + # ASTM F3548-21 + F3548v21DSSQueryOperationalIntentReferences = ( + "astm.f3548.v21.dss.queryOperationalIntentReferences" + ) + F3548v21DSSGetOperationalIntentReference = ( + "astm.f3548.v21.dss.getOperationalIntentReference" + ) + F3548v21DSSCreateOperationalIntentReference = ( + "astm.f3548.v21.dss.createOperationalIntentReference" + ) + F3548v21DSSUpdateOperationalIntentReference = ( + "astm.f3548.v21.dss.updateOperationalIntentReference" + ) + F3548v21DSSDeleteOperationalIntentReference = ( + "astm.f3548.v21.dss.deleteOperationalIntentReference" + ) + F3548v21DSSQueryConstraintReferences = ( + "astm.f3548.v21.dss.queryConstraintReferences" + ) + F3548v21DSSGetConstraintReference = "astm.f3548.v21.dss.getConstraintReference" + F3548v21DSSCreateConstraintReference = ( + "astm.f3548.v21.dss.createConstraintReference" + ) + F3548v21DSSUpdateConstraintReference = ( + "astm.f3548.v21.dss.updateConstraintReference" + ) + F3548v21DSSDeleteConstraintReference = ( + "astm.f3548.v21.dss.deleteConstraintReference" + ) + F3548v21DSSQuerySubscriptions = "astm.f3548.v21.dss.querySubscriptions" + F3548v21DSSGetSubscription = "astm.f3548.v21.dss.getSubscription" + F3548v21DSSCreateSubscription = "astm.f3548.v21.dss.createSubscription" + F3548v21DSSUpdateSubscription = "astm.f3548.v21.dss.updateSubscription" + F3548v21DSSDeleteSubscription = "astm.f3548.v21.dss.deleteSubscription" + F3548v21DSSMakeDssReport = "astm.f3548.v21.dss.makeDssReport" + F3548v21DSSSetUssAvailability = "astm.f3548.v21.uss.setUssAvailability" + F3548v21DSSGetUssAvailability = "astm.f3548.v21.uss.getUssAvailability" + F3548v21USSGetOperationalIntentDetails = ( + "astm.f3548.v21.uss.getOperationalIntentDetails" + ) + F3548v21USSGetOperationalIntentTelemetry = ( + "astm.f3548.v21.uss.getOperationalIntentTelemetry" + ) + F3548v21USSNotifyOperationalIntentDetailsChanged = ( + "astm.f3548.v21.uss.notifyOperationalIntentDetailsChanged" + ) + F3548v21USSGetConstraintDetails = "astm.f3548.v21.uss.getConstraintDetails" + F3548v21USSNotifyConstraintDetailsChanged = ( + "astm.f3548.v21.uss.notifyConstraintDetailsChanged" + ) + F3548v21USSMakeUssReport = "astm.f3548.v21.uss.makeUssReport" + class Query(ImplicitDict): request: RequestDescription @@ -170,6 +237,7 @@ class QueryError(RuntimeError): def __init__(self, msg, queries: Optional[List[Query]] = None): super(RuntimeError, self).__init__(msg) + self.msg = msg self.queries = queries or [] @property @@ -183,18 +251,31 @@ def stacktrace(self) -> str: yaml.add_representer(Query, Representer.represent_dict) +yaml.add_representer(StringBasedDateTime, Representer.represent_str) -def describe_query(resp: requests.Response, initiated_at: datetime.datetime) -> Query: - return Query( + +def describe_query( + resp: requests.Response, + initiated_at: datetime.datetime, + query_type: Optional[QueryType] = None, +) -> Query: + result = Query( request=describe_request(resp.request, initiated_at), response=describe_response(resp), ) + if query_type is not None: + result.query_type = query_type + return result def query_and_describe( - client: Optional[infrastructure.UTMClientSession], verb: str, url: str, **kwargs + client: Optional[infrastructure.UTMClientSession], + verb: str, + url: str, + query_type: Optional[QueryType] = None, + **kwargs, ) -> Query: - """Attempt to perform a query, and the describe the results of that attempt. + """Attempt to perform a query, and then describe the results of that attempt. This function should capture all common problems when attempting to send a query and report the problem in the Query result rather than raising an exception. @@ -203,6 +284,7 @@ def query_and_describe( client: UTMClientSession to use, or None to use a default `requests` Session. verb: HTTP verb to perform at the specified URL. url: URL to query. + query_type: If specified, the known type of query that this is. **kwargs: Any keyword arguments that should be applied to the .request method when invoking it. Returns: @@ -224,7 +306,9 @@ def query_and_describe( for attempt in range(ATTEMPTS): t0 = datetime.datetime.utcnow() try: - return describe_query(client.request(verb, url, **req_kwargs), t0) + return describe_query( + client.request(verb, url, **req_kwargs), t0, query_type + ) except (requests.Timeout, urllib3.exceptions.ReadTimeoutError) as e: failure_message = f"query_and_describe attempt {attempt + 1} from PID {os.getpid()} to {verb} {url} failed with timeout {type(e).__name__}: {str(e)}" logger.warning(failure_message) @@ -245,7 +329,7 @@ def query_and_describe( del req_kwargs["timeout"] req = requests.Request(verb, url, **req_kwargs) prepped_req = client.prepare_request(req) - return Query( + result = Query( request=describe_request(prepped_req, t0), response=ResponseDescription( code=None, @@ -254,3 +338,15 @@ def query_and_describe( reported=StringBasedDateTime(t1), ), ) + if query_type is not None: + result.query_type = query_type + return result + + +def describe_flask_query( + req: flask.Request, res: flask.Response, elapsed_s: float +) -> Query: + return Query( + request=describe_flask_request(req), + response=describe_flask_response(res, elapsed_s), + ) diff --git a/schemas/monitoring/monitorlib/fetch/Query.json b/schemas/monitoring/monitorlib/fetch/Query.json index 26188f8956..6c33dc6ede 100644 --- a/schemas/monitoring/monitorlib/fetch/Query.json +++ b/schemas/monitoring/monitorlib/fetch/Query.json @@ -1,22 +1,66 @@ { + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/fetch/Query.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", + "description": "monitoring.monitorlib.fetch.Query, as defined in monitoring/monitorlib/fetch/__init__.py", "properties": { "$ref": { - "type": "string", - "description": "Path to content that replaces the $ref" + "description": "Path to content that replaces the $ref", + "type": "string" }, - "response": { - "$ref": "ResponseDescription.json" + "query_type": { + "description": "If specified, the recognized type of this query.", + "enum": [ + "astm.f3411.v22a.sp.flights", + "astm.f3411.v19.sp.flights", + "astm.f3411.v22a.sp.flight_details", + "astm.f3411.v19.sp.flight_details", + "astm.f3548.v21.dss.queryOperationalIntentReferences", + "astm.f3548.v21.dss.getOperationalIntentReference", + "astm.f3548.v21.dss.createOperationalIntentReference", + "astm.f3548.v21.dss.updateOperationalIntentReference", + "astm.f3548.v21.dss.deleteOperationalIntentReference", + "astm.f3548.v21.dss.queryConstraintReferences", + "astm.f3548.v21.dss.getConstraintReference", + "astm.f3548.v21.dss.createConstraintReference", + "astm.f3548.v21.dss.updateConstraintReference", + "astm.f3548.v21.dss.deleteConstraintReference", + "astm.f3548.v21.dss.querySubscriptions", + "astm.f3548.v21.dss.getSubscription", + "astm.f3548.v21.dss.createSubscription", + "astm.f3548.v21.dss.updateSubscription", + "astm.f3548.v21.dss.deleteSubscription", + "astm.f3548.v21.dss.makeDssReport", + "astm.f3548.v21.uss.setUssAvailability", + "astm.f3548.v21.uss.getUssAvailability", + "astm.f3548.v21.uss.getOperationalIntentDetails", + "astm.f3548.v21.uss.getOperationalIntentTelemetry", + "astm.f3548.v21.uss.notifyOperationalIntentDetailsChanged", + "astm.f3548.v21.uss.getConstraintDetails", + "astm.f3548.v21.uss.notifyConstraintDetailsChanged", + "astm.f3548.v21.uss.makeUssReport" + ], + "type": [ + "string", + "null" + ] }, "request": { "$ref": "RequestDescription.json" + }, + "response": { + "$ref": "ResponseDescription.json" + }, + "server_id": { + "description": "If specified, identifier of the USS/participant hosting the server involved in this query.", + "type": [ + "string", + "null" + ] } }, - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/monitorlib/fetch/Query.json", - "description": "monitoring.monitorlib.fetch.Query, as defined in monitoring/monitorlib/fetch/__init__.py", "required": [ "request", "response" - ] + ], + "type": "object" } \ No newline at end of file