diff --git a/monitoring/Dockerfile b/monitoring/Dockerfile index 3ca0f72cd6..225320623e 100644 --- a/monitoring/Dockerfile +++ b/monitoring/Dockerfile @@ -22,6 +22,7 @@ RUN rm -rf __pycache__ ADD ./interfaces /app/interfaces ADD ./monitoring /app/monitoring COPY ./monitoring/health_check.sh /app/health_check.sh +RUN chmod 766 /app/health_check.sh WORKDIR /app/monitoring # Additional preparations for uss_qualifier/webapp diff --git a/monitoring/mock_uss/run_locally_tracer.sh b/monitoring/mock_uss/run_locally_tracer.sh index 30df772b0d..33abfaa2e7 100755 --- a/monitoring/mock_uss/run_locally_tracer.sh +++ b/monitoring/mock_uss/run_locally_tracer.sh @@ -20,13 +20,15 @@ POLL='--rid-isa-poll-interval=15 --scd-operation-poll-interval=15 --scd-constrai TRACER_OPTIONS="$AREA $LOGS $MONITOR $POLL" +PORT=${PORT:-8078} AUTH="DummyOAuth(http://host.docker.internal:8085/token,tracer)" -DSS="http://host.docker.internal:8082" +DSS=${MOCK_USS_DSS_URL:-"http://host.docker.internal:8082"} PUBLIC_KEY="/var/test-certs/auth2.pem" AUD=${MOCK_USS_TOKEN_AUDIENCE:-localhost,host.docker.internal} -container_name="mock_uss_tracer" -PORT=8078 +RID_VERSION=${MOCK_USS_RID_VERSION:-"F3411-19"} +CONTAINER_NAME=${MOCK_CONTAINER_NAME:-"mock_uss_tracer"} + BASE_URL="http://${MOCK_USS_TOKEN_AUDIENCE:-host.docker.internal}:${PORT}" if [ "$CI" == "true" ]; then @@ -35,10 +37,10 @@ else docker_args="-it" fi -docker container rm -f ${container_name} || echo "No pre-existing ${container_name} container to remove" +docker container rm -f "${CONTAINER_NAME}" || echo "No pre-existing ${CONTAINER_NAME} container to remove" # shellcheck disable=SC2086 -docker run ${docker_args} --name ${container_name} \ +docker run ${docker_args} --name "${CONTAINER_NAME}" \ -u "$(id -u):$(id -g)" \ -e MOCK_USS_AUTH_SPEC="${AUTH}" \ -e MOCK_USS_DSS_URL="${DSS}" \ @@ -47,6 +49,7 @@ docker run ${docker_args} --name ${container_name} \ -e MOCK_USS_BASE_URL="${BASE_URL}" \ -e MOCK_USS_TRACER_OPTIONS="${TRACER_OPTIONS}" \ -e MOCK_USS_SERVICES="tracer" \ + -e MOCK_USS_RID_VERSION="${RID_VERSION}" \ -p ${PORT}:5000 \ -v "${SCRIPT_DIR}/../../build/test-certs:/var/test-certs:ro" \ -v "$(pwd)/$OUTPUT_DIR:/app/monitoring/mock_uss/$OUTPUT_DIR" \ diff --git a/monitoring/mock_uss/run_locally_tracer_v22a.sh b/monitoring/mock_uss/run_locally_tracer_v22a.sh new file mode 100755 index 0000000000..b648783232 --- /dev/null +++ b/monitoring/mock_uss/run_locally_tracer_v22a.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +if [ -z "${DO_NOT_BUILD_MONITORING}" ]; then + "${SCRIPT_DIR}/../build.sh" || exit 1 +fi + +MOCK_CONTAINER_NAME="mock_uss_tracer_v22a" \ +MOCK_USS_RID_VERSION="F3411-22a" \ +MOCK_USS_DSS_URL="http://host.docker.internal:8082" \ +PORT=8088 \ +"${SCRIPT_DIR}/run_locally_tracer.sh" "$@" \ No newline at end of file diff --git a/monitoring/mock_uss/templates/tracer/base.html b/monitoring/mock_uss/templates/tracer/base.html index 33c6d52b91..be39709f89 100644 --- a/monitoring/mock_uss/templates/tracer/base.html +++ b/monitoring/mock_uss/templates/tracer/base.html @@ -12,7 +12,7 @@
Tracer: Home - RID v19 Flights + RID {{ rid_version }} Flights
diff --git a/monitoring/mock_uss/templates/tracer/rid_poll.html b/monitoring/mock_uss/templates/tracer/rid_poll.html index cd1f6201ec..c4b79a98cc 100644 --- a/monitoring/mock_uss/templates/tracer/rid_poll.html +++ b/monitoring/mock_uss/templates/tracer/rid_poll.html @@ -1,7 +1,7 @@ {% extends "tracer/base.html" %} {% block content %} -
+

diff --git a/monitoring/mock_uss/tracer/config.py b/monitoring/mock_uss/tracer/config.py index 8a73ba20ad..859ce0dff2 100644 --- a/monitoring/mock_uss/tracer/config.py +++ b/monitoring/mock_uss/tracer/config.py @@ -1,9 +1,16 @@ from monitoring.mock_uss import import_environment_variable, require_config_value from monitoring.mock_uss.config import KEY_DSS_URL, KEY_BASE_URL +from monitoring.monitorlib.rid import RIDVersion KEY_TRACER_OPTIONS = "MOCK_USS_TRACER_OPTIONS" +KEY_RID_VERSION = "MOCK_USS_RID_VERSION" import_environment_variable(KEY_TRACER_OPTIONS) require_config_value(KEY_DSS_URL) require_config_value(KEY_BASE_URL) +import_environment_variable( + KEY_RID_VERSION, + default=RIDVersion.f3411_19, + mutator=RIDVersion, +) diff --git a/monitoring/mock_uss/tracer/context.py b/monitoring/mock_uss/tracer/context.py index cb3e789d0d..b427c21c30 100644 --- a/monitoring/mock_uss/tracer/context.py +++ b/monitoring/mock_uss/tracer/context.py @@ -13,6 +13,7 @@ TASK_POLL_OPS, TASK_POLL_CONSTRAINTS, ) +from monitoring.mock_uss.tracer.config import KEY_RID_VERSION from monitoring.monitorlib import ids, versioning from monitoring.monitorlib import fetch import monitoring.monitorlib.fetch.rid @@ -30,6 +31,8 @@ RID_SUBSCRIPTION_ID_CODE = "tracer RID Subscription" SCD_SUBSCRIPTION_ID_CODE = "tracer SCD Subscription" +RID_VERSION = webapp.config[KEY_RID_VERSION] + resources: Optional[ResourceSet] = None @@ -106,9 +109,17 @@ def _subscribe( if base_url.endswith("/"): base_url = base_url[0:-1] if monitor_rid: - _subscribe_rid(resources, base_url + "/tracer/f3411v19") + if RID_VERSION == RIDVersion.f3411_19: + _subscribe_rid(resources, base_url + "/tracer/f3411v19") + elif RID_VERSION == RIDVersion.f3411_22a: + _subscribe_rid(resources, base_url + "/tracer/f3411v22a/v2") + else: + raise NotImplementedError( + f"Cannot subscribe to DSS using RID version {RID_VERSION}" + ) + if monitor_scd: - _subscribe_scd(resources, base_url) + _subscribe_scd(resources, base_url + "/tracer/f3548v21") def _unsubscribe(resources: ResourceSet, monitor_rid: bool, monitor_scd: bool) -> None: @@ -135,8 +146,8 @@ def _subscribe_rid(resources: ResourceSet, uss_base_url: str) -> None: end_time=resources.end_time, uss_base_url=uss_base_url, subscription_id=_rid_subscription_id(), - rid_version=RIDVersion.f3411_19, - utm_client=resources.dss_client, + rid_version=RID_VERSION, + utm_client=resources.dss_clients["rid"], ) resources.logger.log_new(RID_SUBSCRIPTION_KEY, create_result) if not create_result.success: @@ -145,7 +156,7 @@ def _subscribe_rid(resources: ResourceSet, uss_base_url: str) -> None: def _clear_existing_rid_subscription(resources: ResourceSet, suffix: str) -> None: existing_result = fetch.rid.subscription( - _rid_subscription_id(), RIDVersion.f3411_19, resources.dss_client + _rid_subscription_id(), RID_VERSION, resources.dss_clients["rid"] ) logfile = resources.logger.log_new( "{}_{}_get".format(RID_SUBSCRIPTION_KEY, suffix), existing_result @@ -159,8 +170,8 @@ def _clear_existing_rid_subscription(resources: ResourceSet, suffix: str) -> Non del_result = mutate.rid.delete_subscription( subscription_id=_rid_subscription_id(), subscription_version=existing_result.subscription.version, - rid_version=RIDVersion.f3411_19, - utm_client=resources.dss_client, + rid_version=RID_VERSION, + utm_client=resources.dss_clients["rid"], ) logfile = resources.logger.log_new( "{}_{}_del".format(RID_SUBSCRIPTION_KEY, suffix), del_result @@ -183,11 +194,11 @@ def _subscribe_scd(resources: ResourceSet, base_url: str) -> None: _clear_existing_scd_subscription(resources, "old") create_result = mutate.scd.put_subscription( - resources.dss_client, + resources.dss_clients["scd"], resources.area, resources.start_time, resources.end_time, - base_url + "/tracer/f3548v21", + base_url, _scd_subscription_id(), ) logfile = resources.logger.log_new(SCD_SUBSCRIPTION_KEY, create_result) @@ -198,7 +209,9 @@ def _subscribe_scd(resources: ResourceSet, base_url: str) -> None: def _clear_existing_scd_subscription(resources: ResourceSet, suffix: str) -> None: - get_result = fetch.scd.subscription(resources.dss_client, _scd_subscription_id()) + get_result = fetch.scd.subscription( + resources.dss_clients["scd"], _scd_subscription_id() + ) logfile = resources.logger.log_new( "{}_{}_get".format(SCD_SUBSCRIPTION_KEY, suffix), get_result ) @@ -209,7 +222,7 @@ def _clear_existing_scd_subscription(resources: ResourceSet, suffix: str) -> Non if get_result.subscription is not None: del_result = mutate.scd.delete_subscription( - resources.dss_client, + resources.dss_clients["scd"], _scd_subscription_id(), get_result.subscription.version, ) diff --git a/monitoring/mock_uss/tracer/polling.py b/monitoring/mock_uss/tracer/polling.py index 7ef327e790..cefa0eb5d3 100644 --- a/monitoring/mock_uss/tracer/polling.py +++ b/monitoring/mock_uss/tracer/polling.py @@ -13,13 +13,15 @@ def indent(s: str, level: int) -> str: return "\n".join(" " * level + line for line in s.split("\n")) -def poll_rid_isas(resources: ResourceSet, box: s2sphere.LatLngRect) -> Any: +def poll_rid_isas( + resources: ResourceSet, box: s2sphere.LatLngRect, rid_version: RIDVersion +) -> Any: return fetch.rid.isas( box, resources.start_time, resources.end_time, - RIDVersion.f3411_19, - resources.dss_client, + rid_version, + resources.dss_clients["rid"], ) @@ -29,7 +31,7 @@ def poll_scd_operations(resources: ResourceSet) -> Any: str, fetch.scd.FetchedEntity ] = {} return fetch.scd.operations( - resources.dss_client, + resources.dss_clients["scd"], resources.area, resources.start_time, resources.end_time, @@ -41,7 +43,7 @@ def poll_scd_constraints(resources: ResourceSet) -> Any: if "constraints" not in resources.scd_cache: resources.scd_cache["constraints"]: Dict[str, fetch.scd.FetchedEntity] = {} return fetch.scd.constraints( - resources.dss_client, + resources.dss_clients["scd"], resources.area, resources.start_time, resources.end_time, diff --git a/monitoring/mock_uss/tracer/resources.py b/monitoring/mock_uss/tracer/resources.py index 6d1f5dc89d..6b753b9dc5 100644 --- a/monitoring/mock_uss/tracer/resources.py +++ b/monitoring/mock_uss/tracer/resources.py @@ -1,13 +1,16 @@ import argparse import datetime import shlex +from typing import List, Dict import s2sphere from monitoring import mock_uss +from monitoring.mock_uss.config import KEY_DSS_URL +from monitoring.mock_uss.tracer.config import KEY_RID_VERSION +from monitoring.monitorlib.rid import RIDVersion from monitoring.monitorlib import auth, infrastructure, geo from monitoring.mock_uss import webapp, tracer -import monitoring.mock_uss.tracer.config from monitoring.mock_uss.tracer import tracerlog @@ -25,13 +28,13 @@ class ResourceSet(object): def __init__( self, - dss_client: infrastructure.UTMClientSession, + dss_clients: Dict[str, infrastructure.UTMClientSession], area: s2sphere.LatLngRect, logger: tracerlog.Logger, start_time: datetime.datetime, end_time: datetime.datetime, ): - self.dss_client = dss_client + self.dss_clients = dss_clients self.area = area self.logger = logger self.start_time = start_time @@ -102,9 +105,22 @@ def from_arguments(cls, args: argparse.Namespace): adapter: auth.AuthAdapter = auth.make_auth_adapter( webapp.config[mock_uss.config.KEY_AUTH_SPEC] ) - dss_client = infrastructure.UTMClientSession( - webapp.config[mock_uss.config.KEY_DSS_URL], adapter - ) + + if webapp.config[KEY_RID_VERSION] == RIDVersion.f3411_19: + _dss_rid_base_url = webapp.config[KEY_DSS_URL] + elif webapp.config[KEY_RID_VERSION] == RIDVersion.f3411_22a: + _dss_rid_base_url = webapp.config[KEY_DSS_URL] + "/rid/v2" + else: + raise NotImplementedError( + f"Cannot construct DSS base URL using RID version {webapp.config[KEY_RID_VERSION]}" + ) + _dss_scd_base_url = webapp.config[KEY_DSS_URL] + + dss_clients = { + "rid": infrastructure.UTMClientSession(_dss_rid_base_url, adapter), + "scd": infrastructure.UTMClientSession(_dss_scd_base_url, adapter), + } + area: s2sphere.LatLngRect = geo.make_latlng_rect(args.area) start_time = datetime.datetime.fromisoformat(args.start_time) end_time = start_time + datetime.timedelta(hours=args.trace_hours) @@ -122,4 +138,4 @@ def from_arguments(cls, args: argparse.Namespace): if args.output_folder else None ) - return ResourceSet(dss_client, area, logger, start_time, end_time) + return ResourceSet(dss_clients, area, logger, start_time, end_time) diff --git a/monitoring/mock_uss/tracer/routes.py b/monitoring/mock_uss/tracer/routes.py deleted file mode 100644 index d4fd42a80c..0000000000 --- a/monitoring/mock_uss/tracer/routes.py +++ /dev/null @@ -1,350 +0,0 @@ -import datetime -import glob -import os -from typing import Tuple - -import flask -from loguru import logger -from termcolor import colored -import yaml - -from implicitdict import ImplicitDict -from monitoring.monitorlib import fetch, formatting, geo, infrastructure, versioning -from monitoring.monitorlib.fetch import summarize -import monitoring.monitorlib.fetch.rid -import monitoring.monitorlib.fetch.scd -from monitoring.mock_uss import webapp -from . import context -from ...monitorlib.rid import RIDVersion - -RESULT = ("", 204) - - -def _print_time_range(t0: str, t1: str) -> str: - if not t0 and not t1: - return "" - now = datetime.datetime.utcnow() - if t0.endswith("Z"): - t0 = t0[0:-1] - if t1.endswith("Z"): - t1 = t1[0:-1] - try: - t0dt = datetime.datetime.fromisoformat(t0) - now - t1dt = datetime.datetime.fromisoformat(t1) - now - return " {} to {}".format( - formatting.format_timedelta(t0dt), formatting.format_timedelta(t1dt) - ) - except ValueError as e: - return "" - - -@webapp.route( - "/tracer/f3411v19/v1/uss/identification_service_areas/", methods=["POST"] -) -def tracer_rid_v19_isa_notification(id: str) -> Tuple[str, int]: - """Implements RID ISA notification receiver.""" - logger.debug(f"Handling tracer_rid_v19_isa_notification from {os.getpid()}") - req = fetch.describe_flask_request(flask.request) - req["endpoint"] = "identification_service_areas" - log_name = context.resources.logger.log_new("notify_isa", req) - - claims = req.token - owner = claims.get("sub", "") - label = colored("ISA", "cyan") - try: - json = flask.request.json - if json.get("service_area"): - isa = json["service_area"] - owner_body = isa.get("owner") - if owner_body and owner_body != owner: - owner = "{} token|{} body".format(owner, owner_body) - version = isa.get("version", "") - time_range = _print_time_range(isa.get("time_start"), isa.get("time_end")) - logger.info( - "{} {} v{} ({}) updated{} -> {}".format( - label, id, version, owner, time_range, log_name - ) - ) - else: - logger.info("{} {} ({}) deleted -> {}".format(label, id, owner, log_name)) - except ValueError as e: - logger.error( - "{} {} ({}) unable to decode JSON: {} -> {}".format( - label, id, owner, e, log_name - ) - ) - - return RESULT - - -@webapp.route("/tracer/f3548v21/uss/v1/operational_intents", methods=["POST"]) -def tracer_scd_v21_operation_notification() -> Tuple[str, int]: - """Implements SCD Operation notification receiver.""" - logger.debug(f"Handling tracer_scd_v21_operation_notification from {os.getpid()}") - req = fetch.describe_flask_request(flask.request) - req["endpoint"] = "operational_intents" - log_name = context.resources.logger.log_new("notify_op", req) - - claims = req.token - owner = claims.get("sub", "") - label = colored("Operation", "blue") - try: - json = flask.request.json - id = json.get("operational_intent_id", "") - if json.get("operational_intent"): - op = json["operational_intent"] - version = "" - ovn = "" - time_range = "" - if op.get("reference"): - op_ref = op["reference"] - owner_body = op_ref.get("owner") - if owner_body and owner_body != owner: - owner = "{} token|{} body".format(owner, owner_body) - version = op_ref.get("version", version) - ovn = op_ref.get("ovn", ovn) - time_range = _print_time_range( - op_ref.get("time_start", {}).get("value"), - op_ref.get("time_end", {}).get("value"), - ) - state = "" - priority = 0 - if op.get("details"): - op_details = op["details"] - state = op_details.get("state") - priority = op_details.get("priority", 0) - priority_text = str(priority) - logger.info( - "{} {} {} {} v{} ({}) OVN[{}] updated{} -> {}".format( - label, - state, - priority_text, - id, - version, - owner, - ovn, - time_range, - log_name, - ) - ) - else: - logger.info("{} {} ({}) deleted -> {}".format(label, id, owner, log_name)) - except ValueError as e: - logger.error( - "{} ({}) unable to decode JSON: {} -> {}".format(label, owner, e, log_name) - ) - - return RESULT - - -@webapp.route("/tracer/f3548v21/uss/v1/constraints", methods=["POST"]) -def tracer_scd_v21_constraint_notification() -> Tuple[str, int]: - """Implements SCD Constraint notification receiver.""" - logger.debug(f"Handling tracer_scd_v21_constraint_notification from {os.getpid()}") - req = fetch.describe_flask_request(flask.request) - req["endpoint"] = "constraints" - log_name = context.resources.logger.log_new("notify_constraint", req) - - claims = infrastructure.get_token_claims({k: v for k, v in flask.request.headers}) - owner = claims.get("sub", "") - label = colored("Constraint", "magenta") - try: - json = flask.request.json - id = json.get("constraint_id", "") - if json.get("constraint"): - constraint = json["constraint"] - version = "" - ovn = "" - time_range = "" - if constraint.get("reference"): - constraint_ref = constraint["reference"] - owner_body = constraint_ref.get("owner") - if owner_body and owner_body != owner: - owner = "{} token|{} body".format(owner, owner_body) - version = constraint_ref.get("version", version) - ovn = constraint_ref.get("ovn", ovn) - time_range = _print_time_range( - constraint_ref.get("time_start", {}).get("value"), - constraint_ref.get("time_end", {}).get("value"), - ) - type = "" - if constraint.get("details"): - constraint_details = constraint["details"] - type = constraint_details.get("type") - logger.info( - "{} {} {} v{} ({}) OVN[{}] updated{} -> {}".format( - label, type, id, version, owner, ovn, time_range, log_name - ) - ) - else: - logger.info("{} {} ({}) deleted -> {}".format(label, id, owner, log_name)) - except ValueError as e: - logger.error( - "{} ({}) unable to decode JSON: {} -> {}".format(label, owner, e, log_name) - ) - - return RESULT - - -@webapp.route("/tracer/status") -def tracer_status(): - logger.debug(f"Handling tracer_status from {os.getpid()}") - return "Tracer ok {}".format(versioning.get_code_version()) - - -@webapp.route("/tracer/logs") -def tracer_list_logs(): - logger.debug(f"Handling tracer_list_logs from {os.getpid()}") - logs = [ - log - for log in reversed(sorted(os.listdir(context.resources.logger.log_path))) - if log.endswith(".yaml") - ] - kmls = {} - for log in logs: - kml = os.path.join("kml", log[0:-5] + ".kml") - if os.path.exists(os.path.join(context.resources.logger.log_path, kml)): - kmls[log] = kml - response = flask.make_response( - flask.render_template("tracer/logs.html", logs=logs, kmls=kmls) - ) - response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" - response.headers["Pragma"] = "no-cache" - return response - - -def _redact_and_augment_log(obj): - if isinstance(obj, dict): - result = {} - for k, v in obj.items(): - if k.lower() == "authorization" and isinstance(v, str): - result[k] = { - "value": ".".join(v.split(".")[0:-1]) + ".REDACTED", - "claims": infrastructure.get_token_claims(obj), - } - else: - result[k] = _redact_and_augment_log(v) - return result - elif isinstance(obj, str): - return obj - elif isinstance(obj, list): - return [_redact_and_augment_log(item) for item in obj] - else: - return obj - - -@webapp.route("/tracer/logs/") -def tracer_logs(log): - logger.debug(f"Handling tracer_logs from {os.getpid()}") - logfile = os.path.join(context.resources.logger.log_path, log) - if not os.path.exists(logfile): - flask.abort(404) - with open(logfile, "r") as f: - objs = [obj for obj in yaml.full_load_all(f)] - if len(objs) == 1: - obj = objs[0] - else: - obj = {"entries": objs} - - object_type = obj.get("object_type", None) - if object_type == fetch.rid.FetchedISAs.__name__: - obj = { - "summary": summarize.isas(ImplicitDict.parse(obj, fetch.rid.FetchedISAs)), - "details": obj, - } - elif object_type == fetch.scd.FetchedEntities.__name__: - obj = { - "summary": summarize.entities( - ImplicitDict.parse(obj, fetch.scd.FetchedEntities) - ), - "details": obj, - } - elif object_type == fetch.rid.FetchedFlights.__name__: - obj = { - "summary": summarize.flights( - ImplicitDict.parse(obj, fetch.rid.FetchedFlights) - ), - "details": obj, - } - - return flask.render_template( - "tracer/log.html", log=_redact_and_augment_log(obj), title=logfile - ) - - -@webapp.route("/tracer/kml/now.kml") -def tracer_kml_now(): - logger.debug(f"Handling tracer_kml_now from {os.getpid()}") - all_kmls = glob.glob( - os.path.join(context.resources.logger.log_path, "kml", "*.kml") - ) - if not all_kmls: - flask.abort(404, "No KMLs exist") - latest_kml = max(all_kmls, key=os.path.getctime) - return flask.send_file( - latest_kml, - mimetype="application/vnd.google-earth.kml+xml", - attachment_filename="now.kml", - as_attachment=True, - ) - - -@webapp.route("/tracer/kml/") -def tracer_kmls(kml): - logger.debug(f"Handling tracer_kmls from {os.getpid()}") - kmlfile = os.path.join(context.resources.logger.log_path, "kml", kml) - if not os.path.exists(kmlfile): - flask.abort(404) - return flask.send_file( - kmlfile, - mimetype="application/vnd.google-earth.kml+xml", - attachment_filename=kml, - as_attachment=True, - ) - - -@webapp.route("/tracer/f3411v19/rid_poll", methods=["GET"]) -def tracer_rid_v19_get_rid_poll(): - logger.debug(f"Handling tracer_rid_v19_get_rid_poll from {os.getpid()}") - return flask.render_template("tracer/rid_poll.html") - - -@webapp.route("/tracer/f3411v19/rid_poll", methods=["POST"]) -def tracer_rid_v19_request_rid_poll(): - logger.debug(f"Handling tracer_rid_v19_request_rid_poll from {os.getpid()}") - if "area" not in flask.request.form: - flask.abort(400, "Missing area") - - try: - area = geo.make_latlng_rect(flask.request.form["area"]) - except ValueError as e: - flask.abort(400, str(e)) - return - - flights_result = fetch.rid.all_flights( - area, - flask.request.form.get("include_recent_positions"), - flask.request.form.get("get_details"), - RIDVersion.f3411_19, - context.resources.dss_client, - enhanced_details=flask.request.form.get("enhanced_details"), - ) - log_name = context.resources.logger.log_new( - "clientrequest_getflights", flights_result - ) - return flask.redirect(flask.url_for("tracer_logs", log=log_name)) - - -@webapp.route("/tracer/", methods=["GET", "PUT", "POST", "DELETE"]) -def tracer_catch_all(u_path) -> Tuple[str, int]: - logger.debug(f"Handling tracer_catch_all from {os.getpid()}") - req = fetch.describe_flask_request(flask.request) - req["endpoint"] = "catch_all" - log_name = context.resources.logger.log_new("uss_badroute", req) - - claims = req.token - owner = claims.get("sub", "") - label = colored("Bad route", "red") - logger.error("{} to {} ({}): {}".format(label, u_path, owner, log_name)) - - return f"Path is not a supported endpoint: {u_path}", 404 diff --git a/monitoring/mock_uss/tracer/routes/__init__.py b/monitoring/mock_uss/tracer/routes/__init__.py new file mode 100644 index 0000000000..07dd20fa96 --- /dev/null +++ b/monitoring/mock_uss/tracer/routes/__init__.py @@ -0,0 +1,36 @@ +import os +from typing import Tuple + +import flask +from loguru import logger +from termcolor import colored + +from monitoring.mock_uss import webapp +from monitoring.monitorlib import fetch, versioning +from .. import context + + +@webapp.route("/tracer/status") +def tracer_status(): + logger.debug(f"Handling tracer_status from {os.getpid()}") + return "Tracer ok {}".format(versioning.get_code_version()) + + +from monitoring.mock_uss.tracer.routes import views +from monitoring.mock_uss.tracer.routes import scd +from monitoring.mock_uss.tracer.routes import rid + + +@webapp.route("/tracer/", methods=["GET", "PUT", "POST", "DELETE"]) +def tracer_catch_all(u_path) -> Tuple[str, int]: + logger.debug(f"Handling tracer_catch_all from {os.getpid()}") + req = fetch.describe_flask_request(flask.request) + req["endpoint"] = "catch_all" + log_name = context.resources.logger.log_new("uss_badroute", req) + + claims = req.token + owner = claims.get("sub", "") + label = colored("Bad route", "red") + logger.error("{} to {} ({}): {}".format(label, u_path, owner, log_name)) + + return f"Path is not a supported endpoint: {u_path}", 404 diff --git a/monitoring/mock_uss/tracer/routes/rid.py b/monitoring/mock_uss/tracer/routes/rid.py new file mode 100644 index 0000000000..f2be2dcea7 --- /dev/null +++ b/monitoring/mock_uss/tracer/routes/rid.py @@ -0,0 +1,87 @@ +import os +from typing import Tuple + +import flask +from loguru import logger +from termcolor import colored + +from implicitdict import ImplicitDict +from monitoring.mock_uss import webapp +from monitoring.monitorlib import fetch +from monitoring.monitorlib.rid import RIDVersion +from uas_standards.astm.f3411.v19.api import ( + PutIdentificationServiceAreaNotificationParameters as PutIdentificationServiceAreaNotificationParametersV19, +) +from uas_standards.astm.f3411.v22a.api import ( + PutIdentificationServiceAreaNotificationParameters as PutIdentificationServiceAreaNotificationParametersV22a, +) +from .. import context +from ..config import KEY_RID_VERSION +from ..template import _print_time_range + +RESULT = ("", 204) +RID_VERSION = webapp.config[KEY_RID_VERSION] + + +if RID_VERSION == RIDVersion.f3411_19: + path = "/tracer/f3411v19/v1/uss/identification_service_areas/" +elif RID_VERSION == RIDVersion.f3411_22a: + path = "/tracer/f3411v22a/v2/uss/identification_service_areas/" +else: + raise NotImplementedError( + f"Unsupported RID Version {RID_VERSION}. No routes mounted for RID notifications." + ) + + +@webapp.route(path, methods=["POST"]) +def tracer_rid_isa_notification(id: str) -> Tuple[str, int]: + """Implements RID ISA notification receiver.""" + logger.debug(f"Handling tracer_rid_isa_notification from {os.getpid()}") + req = fetch.describe_flask_request(flask.request) + req["endpoint"] = "identification_service_areas" + log_name = context.resources.logger.log_new("notify_isa", req) + + claims = req.token + owner = claims.get("sub", "") + label = colored("ISA", "cyan") + try: + json = flask.request.json + if json is None: + raise ValueError("Request did not contain a JSON payload") + + # TODO: Use mutate.rid.ISAChangeNotification when fully implemented. See https://github.com/interuss/monitoring/pull/123/files/553f46b374623e3734634bb277548e06a2457cd6#r1255701016 + if RID_VERSION == RIDVersion.f3411_19: + notification = ImplicitDict.parse( + json, PutIdentificationServiceAreaNotificationParametersV19 + ) + if RID_VERSION == RIDVersion.f3411_22a: + notification = ImplicitDict.parse( + json, PutIdentificationServiceAreaNotificationParametersV22a + ) + + if notification.get("service_area", None): + isa = notification.service_area + owner_body = isa.owner + if owner_body and owner_body != owner: + owner = f"{owner} token|{owner_body} body" + version = isa.version if isa.version else "" + if RID_VERSION == RIDVersion.f3411_19: + time_range = _print_time_range(isa.time_start, isa.time_end) + elif RID_VERSION == RIDVersion.f3411_22a: + time_range = _print_time_range(isa.time_start.value, isa.time_end.value) + else: + raise NotImplementedError( + f"Unsupported RID Version {RID_VERSION}. Unable to retrieve time range from isa response {isa}." + ) + + logger.info( + f"{label} {id} v{version} ({owner}) updated {time_range} -> {log_name}" + ) + else: + logger.info(f"{label} {id} ({owner}) deleted -> {log_name}") + except ValueError as err: + logger.error( + f"{label} {id} ({owner}) unable to decode JSON: {err} -> {log_name}" + ) + + return RESULT diff --git a/monitoring/mock_uss/tracer/routes/scd.py b/monitoring/mock_uss/tracer/routes/scd.py new file mode 100644 index 0000000000..fd4a1a5edc --- /dev/null +++ b/monitoring/mock_uss/tracer/routes/scd.py @@ -0,0 +1,122 @@ +import os +from typing import Tuple + +import flask +from loguru import logger +from termcolor import colored + +from monitoring.mock_uss import webapp +from monitoring.monitorlib import fetch, infrastructure +from . import context +from ..template import _print_time_range + +RESULT = ("", 204) + + +@webapp.route("/tracer/f3548v21/uss/v1/operational_intents", methods=["POST"]) +def tracer_scd_v21_operation_notification() -> Tuple[str, int]: + """Implements SCD Operation notification receiver.""" + logger.debug(f"Handling tracer_scd_v21_operation_notification from {os.getpid()}") + req = fetch.describe_flask_request(flask.request) + req["endpoint"] = "operational_intents" + log_name = context.resources.logger.log_new("notify_op", req) + + claims = req.token + owner = claims.get("sub", "") + label = colored("Operation", "blue") + try: + json = flask.request.json + id = json.get("operational_intent_id", "") + if json.get("operational_intent"): + op = json["operational_intent"] + version = "" + ovn = "" + time_range = "" + if op.get("reference"): + op_ref = op["reference"] + owner_body = op_ref.get("owner") + if owner_body and owner_body != owner: + owner = "{} token|{} body".format(owner, owner_body) + version = op_ref.get("version", version) + ovn = op_ref.get("ovn", ovn) + time_range = _print_time_range( + op_ref.get("time_start", {}).get("value"), + op_ref.get("time_end", {}).get("value"), + ) + state = "" + priority = 0 + if op.get("details"): + op_details = op["details"] + state = op_details.get("state") + priority = op_details.get("priority", 0) + priority_text = str(priority) + logger.info( + "{} {} {} {} v{} ({}) OVN[{}] updated{} -> {}".format( + label, + state, + priority_text, + id, + version, + owner, + ovn, + time_range, + log_name, + ) + ) + else: + logger.info("{} {} ({}) deleted -> {}".format(label, id, owner, log_name)) + except ValueError as e: + logger.error( + "{} ({}) unable to decode JSON: {} -> {}".format(label, owner, e, log_name) + ) + + return RESULT + + +@webapp.route("/tracer/f3548v21/uss/v1/constraints", methods=["POST"]) +def tracer_scd_v21_constraint_notification() -> Tuple[str, int]: + """Implements SCD Constraint notification receiver.""" + logger.debug(f"Handling tracer_scd_v21_constraint_notification from {os.getpid()}") + req = fetch.describe_flask_request(flask.request) + req["endpoint"] = "constraints" + log_name = context.resources.logger.log_new("notify_constraint", req) + + claims = infrastructure.get_token_claims({k: v for k, v in flask.request.headers}) + owner = claims.get("sub", "") + label = colored("Constraint", "magenta") + try: + json = flask.request.json + id = json.get("constraint_id", "") + if json.get("constraint"): + constraint = json["constraint"] + version = "" + ovn = "" + time_range = "" + if constraint.get("reference"): + constraint_ref = constraint["reference"] + owner_body = constraint_ref.get("owner") + if owner_body and owner_body != owner: + owner = "{} token|{} body".format(owner, owner_body) + version = constraint_ref.get("version", version) + ovn = constraint_ref.get("ovn", ovn) + time_range = _print_time_range( + constraint_ref.get("time_start", {}).get("value"), + constraint_ref.get("time_end", {}).get("value"), + ) + type = "" + if constraint.get("details"): + constraint_details = constraint["details"] + type = constraint_details.get("type") + logger.info( + "{} {} {} v{} ({}) OVN[{}] updated{} -> {}".format( + label, type, id, version, owner, ovn, time_range, log_name + ) + ) + else: + logger.info("{} {} ({}) deleted -> {}".format(label, id, owner, log_name)) + except ValueError as e: + logger.error( + "{} ({}) unable to decode JSON: {} -> {}".format(label, owner, e, log_name) + ) + + return RESULT diff --git a/monitoring/mock_uss/tracer/routes/views.py b/monitoring/mock_uss/tracer/routes/views.py new file mode 100644 index 0000000000..ba5fe9bc18 --- /dev/null +++ b/monitoring/mock_uss/tracer/routes/views.py @@ -0,0 +1,168 @@ +import glob +import os + +import flask +import monitoring.monitorlib.fetch.rid +import monitoring.monitorlib.fetch.scd +import yaml +from loguru import logger +from monitoring.mock_uss import webapp +from monitoring.monitorlib import fetch, geo, infrastructure +from monitoring.monitorlib.fetch import summarize + +from implicitdict import ImplicitDict +from ..config import KEY_RID_VERSION + +from .. import context + +RID_VERSION = webapp.config[KEY_RID_VERSION] + + +@webapp.route("/tracer/logs") +def tracer_list_logs(): + logger.debug(f"Handling tracer_list_logs from {os.getpid()}") + logs = [ + log + for log in reversed(sorted(os.listdir(context.resources.logger.log_path))) + if log.endswith(".yaml") + ] + kmls = {} + for log in logs: + kml = os.path.join("kml", log[0:-5] + ".kml") + if os.path.exists(os.path.join(context.resources.logger.log_path, kml)): + kmls[log] = kml + response = flask.make_response( + flask.render_template( + "tracer/logs.html", logs=logs, kmls=kmls, rid_version=RID_VERSION.short_name + ) + ) + response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + response.headers["Pragma"] = "no-cache" + return response + + +def _redact_and_augment_log(obj): + if isinstance(obj, dict): + result = {} + for k, v in obj.items(): + if k.lower() == "authorization" and isinstance(v, str): + result[k] = { + "value": ".".join(v.split(".")[0:-1]) + ".REDACTED", + "claims": infrastructure.get_token_claims(obj), + } + else: + result[k] = _redact_and_augment_log(v) + return result + elif isinstance(obj, str): + return obj + elif isinstance(obj, list): + return [_redact_and_augment_log(item) for item in obj] + else: + return obj + + +@webapp.route("/tracer/logs/") +def tracer_logs(log): + logger.debug(f"Handling tracer_logs from {os.getpid()}") + logfile = os.path.join(context.resources.logger.log_path, log) + if not os.path.exists(logfile): + flask.abort(404) + with open(logfile, "r") as f: + objs = [obj for obj in yaml.full_load_all(f)] + if len(objs) == 1: + obj = objs[0] + else: + obj = {"entries": objs} + + object_type = obj.get("object_type", None) + if object_type == fetch.rid.FetchedISAs.__name__: + obj = { + "summary": summarize.isas(ImplicitDict.parse(obj, fetch.rid.FetchedISAs)), + "details": obj, + } + elif object_type == fetch.scd.FetchedEntities.__name__: + obj = { + "summary": summarize.entities( + ImplicitDict.parse(obj, fetch.scd.FetchedEntities) + ), + "details": obj, + } + elif object_type == fetch.rid.FetchedFlights.__name__: + obj = { + "summary": summarize.flights( + ImplicitDict.parse(obj, fetch.rid.FetchedFlights) + ), + "details": obj, + } + + return flask.render_template( + "tracer/log.html", + log=_redact_and_augment_log(obj), + title=logfile, + rid_version=RID_VERSION.short_name, + ) + + +@webapp.route("/tracer/kml/now.kml") +def tracer_kml_now(): + logger.debug(f"Handling tracer_kml_now from {os.getpid()}") + all_kmls = glob.glob( + os.path.join(context.resources.logger.log_path, "kml", "*.kml") + ) + if not all_kmls: + flask.abort(404, "No KMLs exist") + latest_kml = max(all_kmls, key=os.path.getctime) + return flask.send_file( + latest_kml, + mimetype="application/vnd.google-earth.kml+xml", + attachment_filename="now.kml", + as_attachment=True, + ) + + +@webapp.route("/tracer/kml/") +def tracer_kmls(kml): + logger.debug(f"Handling tracer_kmls from {os.getpid()}") + kmlfile = os.path.join(context.resources.logger.log_path, "kml", kml) + if not os.path.exists(kmlfile): + flask.abort(404) + return flask.send_file( + kmlfile, + mimetype="application/vnd.google-earth.kml+xml", + attachment_filename=kml, + as_attachment=True, + ) + + +@webapp.route("/tracer/rid/poll", methods=["GET"]) +def tracer_rid_get_poll(): + logger.debug(f"Handling tracer_rid_get_poll from {os.getpid()}") + return flask.render_template( + "tracer/rid_poll.html", rid_version=RID_VERSION.short_name + ) + + +@webapp.route("/tracer/rid/poll", methods=["POST"]) +def tracer_rid_request_poll(): + logger.debug(f"Handling tracer_rid_request_poll from {os.getpid()}") + if "area" not in flask.request.form: + flask.abort(400, "Missing area") + + try: + area = geo.make_latlng_rect(flask.request.form["area"]) + except ValueError as err: + flask.abort(400, str(err)) + return + + flights_result = fetch.rid.all_flights( + area, + flask.request.form.get("include_recent_positions"), + flask.request.form.get("get_details"), + RID_VERSION, + context.resources.dss_clients["rid"], + enhanced_details=flask.request.form.get("enhanced_details"), + ) + log_name = context.resources.logger.log_new( + "clientrequest_getflights", flights_result + ) + return flask.redirect(flask.url_for("tracer_logs", log=log_name)) diff --git a/monitoring/mock_uss/tracer/template.py b/monitoring/mock_uss/tracer/template.py new file mode 100644 index 0000000000..7f242e6342 --- /dev/null +++ b/monitoring/mock_uss/tracer/template.py @@ -0,0 +1,20 @@ +import datetime +from monitoring.monitorlib import formatting + + +def _print_time_range(t0: str, t1: str) -> str: + if not t0 and not t1: + return "" + now = datetime.datetime.utcnow() + if t0.endswith("Z"): + t0 = t0[0:-1] + if t1.endswith("Z"): + t1 = t1[0:-1] + try: + t0dt = datetime.datetime.fromisoformat(t0) - now + t1dt = datetime.datetime.fromisoformat(t1) - now + return " {} to {}".format( + formatting.format_timedelta(t0dt), formatting.format_timedelta(t1dt) + ) + except ValueError as e: + return "" diff --git a/monitoring/mock_uss/tracer/tracer_poll.py b/monitoring/mock_uss/tracer/tracer_poll.py index f8841044d4..6fe172aa7c 100755 --- a/monitoring/mock_uss/tracer/tracer_poll.py +++ b/monitoring/mock_uss/tracer/tracer_poll.py @@ -4,6 +4,8 @@ from typing import Optional from implicitdict import ImplicitDict + +from monitoring.mock_uss.tracer.config import KEY_RID_VERSION from monitoring.monitorlib import versioning from monitoring.mock_uss import webapp from monitoring.mock_uss.tracer import diff, polling @@ -17,6 +19,8 @@ TASK_POLL_OPS = "tracer poll ops" TASK_POLL_CONSTRAINTS = "tracer poll constraints" +RID_VERSION = webapp.config[KEY_RID_VERSION] + class PollingStatus(ImplicitDict): started: bool = False @@ -71,7 +75,7 @@ def poll_isas() -> datetime: log_name = "poll_isas" t0 = datetime.datetime.utcnow() - result = polling.poll_rid_isas(resources, resources.area) + result = polling.poll_rid_isas(resources, resources.area, RID_VERSION) t1 = datetime.datetime.utcnow() log_new = False diff --git a/monitoring/monitorlib/fetch/rid.py b/monitoring/monitorlib/fetch/rid.py index a902dce0f9..9a4d824ce0 100644 --- a/monitoring/monitorlib/fetch/rid.py +++ b/monitoring/monitorlib/fetch/rid.py @@ -432,7 +432,7 @@ def has_different_content_than(self, other: Any) -> bool: if self.rid_version == RIDVersion.f3411_19: return self._v19_response != other._v19_response elif self.rid_version == RIDVersion.f3411_22a: - return self._v22_response != other._v22_response + return self._v22a_response != other._v22a_response else: raise NotImplementedError( f"Cannot compare ISAs using RID version {self.rid_version}" diff --git a/monitoring/monitorlib/rid.py b/monitoring/monitorlib/rid.py index 93b55ef664..ef3572adc2 100644 --- a/monitoring/monitorlib/rid.py +++ b/monitoring/monitorlib/rid.py @@ -90,3 +90,12 @@ def min_cluster_size_percent(self) -> float: return v22a.constants.NetMinClusterSizePercent else: raise ValueError("Unsupported RID version '{}'".format(self)) + + @property + def short_name(self) -> str: + if self == RIDVersion.f3411_19: + return "v19" + elif self == RIDVersion.f3411_22a: + return "v22a" + else: + return "unknown"