Merge remote-tracking branch 'interuss/main' into optimize-rid-tests

Makefile

@@ -14,7 +14,7 @@ format: json-schema
 	cd monitoring && make format
 
 .PHONY: lint
-lint:
+lint: shell-lint
 	cd monitoring && make lint
 	cd schemas && make lint
 
@@ -35,8 +35,7 @@ validate-uss-qualifier-docs:
 
 .PHONY: shell-lint
 shell-lint:
-	echo "===== Checking DSS shell lint except monitoring =====" && find . -name '*.sh' | grep -v '^./interfaces/astm-utm' | grep -v '^./monitoring' | xargs docker run --rm -v "$(CURDIR):/monitoring" -w /monitoring koalaman/shellcheck
-	cd monitoring && make shell-lint
+	find . -name '*.sh' | grep -v '^./interfaces' | xargs docker run --rm -v "$(CURDIR):/monitoring" -w /monitoring koalaman/shellcheck
 
 .PHONY: json-schema
 json-schema:

build/dev/extract_json_field.py

@@ -18,5 +18,7 @@
         try:
             obj = obj[field]
         except KeyError:
-            raise ValueError(f"Could not find field '{field}' in '{sys.argv[1]}' for {sys.argv[2]}; available keys: {list(obj.keys())}")
+            raise ValueError(
+                f"Could not find field '{field}' in '{sys.argv[1]}' for {sys.argv[2]}; available keys: {list(obj.keys())}"
+            )
 print(obj)

monitoring/Makefile

@@ -12,12 +12,6 @@ python-lint:
 	cd monitorlib && make python-lint
 	cd prober && make python-lint
 
-.PHONY: shell-lint
-shell-lint:
-	cd uss_qualifier && make shell-lint
-	cd mock_uss && make shell-lint
-	cd prober && make shell-lint
-
 .PHONY: format
 format:
 	cd uss_qualifier && make format

monitoring/atproxy/config.py

@@ -5,34 +5,35 @@
 
 from monitoring.monitorlib import auth_validation
 
-ENV_KEY_PREFIX = 'ATPROXY'
-ENV_KEY_PUBLIC_KEY = '{}_PUBLIC_KEY'.format(ENV_KEY_PREFIX)
-ENV_KEY_TOKEN_AUDIENCE = '{}_TOKEN_AUDIENCE'.format(ENV_KEY_PREFIX)
-ENV_KEY_CLIENT_BASIC_AUTH = '{}_CLIENT_BASIC_AUTH'.format(ENV_KEY_PREFIX)
-ENV_KEY_QUERY_TIMEOUT = '{}_QUERY_TIMEOUT'.format(ENV_KEY_PREFIX)
+ENV_KEY_PREFIX = "ATPROXY"
+ENV_KEY_PUBLIC_KEY = "{}_PUBLIC_KEY".format(ENV_KEY_PREFIX)
+ENV_KEY_TOKEN_AUDIENCE = "{}_TOKEN_AUDIENCE".format(ENV_KEY_PREFIX)
+ENV_KEY_CLIENT_BASIC_AUTH = "{}_CLIENT_BASIC_AUTH".format(ENV_KEY_PREFIX)
+ENV_KEY_QUERY_TIMEOUT = "{}_QUERY_TIMEOUT".format(ENV_KEY_PREFIX)
 
 # These keys map to entries in the Config class
-KEY_TOKEN_PUBLIC_KEY = 'TOKEN_PUBLIC_KEY'
-KEY_TOKEN_AUDIENCE = 'TOKEN_AUDIENCE'
-KEY_CLIENT_BASIC_AUTH = 'CLIENT_BASIC_AUTH'
-KEY_QUERY_TIMEOUT = 'QUERY_TIMEOUT'
+KEY_TOKEN_PUBLIC_KEY = "TOKEN_PUBLIC_KEY"
+KEY_TOKEN_AUDIENCE = "TOKEN_AUDIENCE"
+KEY_CLIENT_BASIC_AUTH = "CLIENT_BASIC_AUTH"
+KEY_QUERY_TIMEOUT = "QUERY_TIMEOUT"
 
-KEY_CODE_VERSION = 'MONITORING_VERSION'
+KEY_CODE_VERSION = "MONITORING_VERSION"
 
-workspace_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'workspace')
+workspace_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "workspace")
 
 
 class Config(object):
     TOKEN_PUBLIC_KEY = auth_validation.fix_key(
-        os.environ.get(ENV_KEY_PUBLIC_KEY, '')).encode('utf-8')
-    TOKEN_AUDIENCE = os.environ.get(ENV_KEY_TOKEN_AUDIENCE, '')
+        os.environ.get(ENV_KEY_PUBLIC_KEY, "")
+    ).encode("utf-8")
+    TOKEN_AUDIENCE = os.environ.get(ENV_KEY_TOKEN_AUDIENCE, "")
     CLIENT_BASIC_AUTH = os.environ[ENV_KEY_CLIENT_BASIC_AUTH]
     QUERY_TIMEOUT = float(os.environ.get(ENV_KEY_QUERY_TIMEOUT, "59"))
-    CODE_VERSION = os.environ.get(KEY_CODE_VERSION, 'Unknown')
+    CODE_VERSION = os.environ.get(KEY_CODE_VERSION, "Unknown")
 
 
 def get_users(basic_auth: str) -> Dict[str, str]:
-    user_pass = [v.strip() for v in basic_auth.split(':')]
+    user_pass = [v.strip() for v in basic_auth.split(":")]
     if len(user_pass) != 2:
         raise ValueError('Expected "username:password", got "{}"'.format(basic_auth))
     return {user_pass[0]: generate_password_hash(user_pass[1])}

monitoring/atproxy/database.py

@@ -10,9 +10,10 @@
 # --- All queries ---
 class QueryState(str, enum.Enum):
     """Whether a query is being handled, or has already been handled."""
-    Queued = 'Queued'
-    BeingHandled = 'BeingHandled'
-    Complete = 'Complete'
+
+    Queued = "Queued"
+    BeingHandled = "BeingHandled"
+    Complete = "Complete"
 
 
 class Query(ImplicitDict):
@@ -51,4 +52,5 @@ class Database(ImplicitDict):
 
 db = SynchronizedValue(
     Database(),
-    decoder=lambda b: ImplicitDict.parse(json.loads(b.decode('utf-8')), Database))
+    decoder=lambda b: ImplicitDict.parse(json.loads(b.decode("utf-8")), Database),
+)

monitoring/atproxy/gunicorn.conf.py

@@ -8,14 +8,29 @@
 
 def pre_request(worker: Worker, req: Request):
     """gunicorn server hook called just before a worker processes the request."""
-    logger.debug("gunicorn pre_request from worker {} (OS PID {}): {} {}", worker.pid, os.getpid(), req.method, req.path)
+    logger.debug(
+        "gunicorn pre_request from worker {} (OS PID {}): {} {}",
+        worker.pid,
+        os.getpid(),
+        req.method,
+        req.path,
+    )
 
 
 def post_request(worker: Worker, req: Request, environ: dict, resp: Response):
     """gunicorn server hook called after a worker processes the request."""
-    logger.debug("gunicorn post_request from worker {} (OS PID {}): {} {} -> {}", worker.pid, os.getpid(), req.method, req.path, resp.status_code)
+    logger.debug(
+        "gunicorn post_request from worker {} (OS PID {}): {} {} -> {}",
+        worker.pid,
+        os.getpid(),
+        req.method,
+        req.path,
+        resp.status_code,
+    )
 
 
 def worker_abort(worker: Worker):
     """gunicorn server hook called when a worker received the SIGABRT signal."""
-    logger.debug("gunicorn worker_abort from worker {} (OS PID {})", worker.pid, os.getpid())
+    logger.debug(
+        "gunicorn worker_abort from worker {} (OS PID {})", worker.pid, os.getpid()
+    )

monitoring/atproxy/handling.py

@@ -51,29 +51,47 @@ def fulfill_query(req: ImplicitDict, timeout: timedelta) -> Tuple[str, int]:
     t_start = datetime.utcnow()
     query = Query(type=req.request_type_name(), request=req)
     id = str(uuid.uuid4())
-    logger.debug('Attempting to fulfill {} query {} from worker {}', query.type, id, os.getpid())
+    logger.debug(
+        "Attempting to fulfill {} query {} from worker {}", query.type, id, os.getpid()
+    )
 
     # Add query to be handled to the set of handleable queries
     with db as tx:
         tx.queries[id] = query
-        logger.debug('Added {} query {} to handler queue'.format(query.type, id))
+        logger.debug("Added {} query {} to handler queue".format(query.type, id))
 
     # Frequently check if the query has been fulfilled
     while datetime.utcnow() < t_start + timeout:
         time.sleep(0.1)
         with db as tx:
             if tx.queries[id].state == QueryState.Complete:
                 # Query was successfully fulfilled; return the result
-                logger.debug('Fulfilling {} query {}'.format(query.type, id))
+                logger.debug("Fulfilling {} query {}".format(query.type, id))
                 query = tx.queries.pop(id)
-                logger.debug('Fulfilled {} query {} with {} from worker {}', query.type, id, query.return_code, os.getpid())
+                logger.debug(
+                    "Fulfilled {} query {} with {} from worker {}",
+                    query.type,
+                    id,
+                    query.return_code,
+                    os.getpid(),
+                )
                 if query.response is not None:
                     return flask.jsonify(query.response), query.return_code
                 else:
-                    return '', query.return_code
+                    return "", query.return_code
 
     # Time expired; remove request from queue and indicate error
     with db as tx:
         tx.queries.pop(id)
-    logger.debug('Failed to fulfill {} query {} in time (backend handler did not provide a response) from worker {}', query.type, id, os.getpid())
-    return flask.jsonify({'message': 'Backend handler did not respond within the alotted time'}), 500
+    logger.debug(
+        "Failed to fulfill {} query {} in time (backend handler did not provide a response) from worker {}",
+        query.type,
+        id,
+        os.getpid(),
+    )
+    return (
+        flask.jsonify(
+            {"message": "Backend handler did not respond within the alotted time"}
+        ),
+        500,
+    )

monitoring/atproxy/oauth.py

@@ -5,4 +5,5 @@
 
 requires_scope = auth_validation.requires_scope_decorator(
     webapp.config.get(config.KEY_TOKEN_PUBLIC_KEY),
-    webapp.config.get(config.KEY_TOKEN_AUDIENCE))
+    webapp.config.get(config.KEY_TOKEN_AUDIENCE),
+)

monitoring/atproxy/requests.py

@@ -3,7 +3,8 @@
 from monitoring.monitorlib.rid_automated_testing import injection_api
 from implicitdict import ImplicitDict
 from uas_standards.interuss.automated_testing.scd.v1.api import (
-    ClearAreaRequest, InjectFlightRequest,
+    ClearAreaRequest,
+    InjectFlightRequest,
 )
 
 
@@ -19,7 +20,13 @@ class RequestType(str, Enum):
     SCD_CreateClearAreaRequest = "scd.createClearAreaRequest"
 
 
-SCD_REQUESTS = {RequestType.SCD_GetStatus, RequestType.SCD_GetCapabilities, RequestType.SCD_PutFlight, RequestType.SCD_DeleteFlight, RequestType.SCD_CreateClearAreaRequest}
+SCD_REQUESTS = {
+    RequestType.SCD_GetStatus,
+    RequestType.SCD_GetCapabilities,
+    RequestType.SCD_PutFlight,
+    RequestType.SCD_DeleteFlight,
+    RequestType.SCD_CreateClearAreaRequest,
+}
 
 
 # Each request descriptor in this file is expected to implement a static

monitoring/atproxy/routes.py

@@ -9,20 +9,20 @@
 from .app import webapp, basic_auth, users
 
 
-@webapp.route('/')
+@webapp.route("/")
 def root() -> Tuple[str, int]:
-    return 'ok', 200
+    return "ok", 200
 
 
-@webapp.route('/favicon.ico')
+@webapp.route("/favicon.ico")
 def favicon():
-  flask.abort(404)
+    flask.abort(404)
 
 
-@webapp.route('/status')
+@webapp.route("/status")
 @basic_auth.login_required
 def status():
-    return 'atproxy ok {}'.format(versioning.get_code_version())
+    return "atproxy ok {}".format(versioning.get_code_version())
 
 
 @webapp.errorhandler(Exception)
@@ -31,17 +31,31 @@ def handle_exception(e):
     if isinstance(e, HTTPException):
         return e
     elif isinstance(e, auth_validation.InvalidScopeError):
-        return flask.jsonify({
-            'message': 'Invalid scope; expected one of {%s}, but received only {%s}' % (' '.join(e.permitted_scopes),
-                                                                                        ' '.join(e.provided_scopes))}), 403
+        return (
+            flask.jsonify(
+                {
+                    "message": "Invalid scope; expected one of {%s}, but received only {%s}"
+                    % (" ".join(e.permitted_scopes), " ".join(e.provided_scopes))
+                }
+            ),
+            403,
+        )
     elif isinstance(e, auth_validation.InvalidAccessTokenError):
-        return flask.jsonify({'message': e.message}), 401
+        return flask.jsonify({"message": e.message}), 401
     elif isinstance(e, auth_validation.ConfigurationError):
-        return flask.jsonify({'message': 'Auth validation configuration error: ' + e.message}), 500
+        return (
+            flask.jsonify(
+                {"message": "Auth validation configuration error: " + e.message}
+            ),
+            500,
+        )
     elif isinstance(e, ValueError):
-        return flask.jsonify({'message': str(e)}), 400
+        return flask.jsonify({"message": str(e)}), 400
 
-    return flask.jsonify({'message': 'Unhandled {}: {}'.format(type(e).__name__, str(e))}), 500
+    return (
+        flask.jsonify({"message": "Unhandled {}: {}".format(type(e).__name__, str(e))}),
+        500,
+    )
 
 
 @basic_auth.verify_password

monitoring/atproxy/routes_handler.py

@@ -12,53 +12,64 @@
 from .handling import ListQueriesResponse, PendingRequest, PutQueryRequest
 
 
-@webapp.route('/handler/queries', methods=['GET'])
+@webapp.route("/handler/queries", methods=["GET"])
 @basic_auth.login_required
 def list_queries() -> Tuple[str, int]:
     """Lists outstanding queries to be handled.
 
     See ListQueriesResponse for response body schema.
     """
     t_start = datetime.utcnow()
-    logger.debug('Handler requesting queries from worker {}', os.getpid())
+    logger.debug("Handler requesting queries from worker {}", os.getpid())
     max_timeout = timedelta(seconds=5)
     while datetime.utcnow() < t_start + max_timeout:
         with db as tx:
-            response = ListQueriesResponse(requests=[
-                PendingRequest(id=id, type=q.type, request=q.request)
-                for id, q in tx.queries.items()
-                if q.state == QueryState.Queued])
+            response = ListQueriesResponse(
+                requests=[
+                    PendingRequest(id=id, type=q.type, request=q.request)
+                    for id, q in tx.queries.items()
+                    if q.state == QueryState.Queued
+                ]
+            )
             if response.requests:
-                logger.debug('Provided handler {} queries'.format(len(response.requests)))
+                logger.debug(
+                    "Provided handler {} queries".format(len(response.requests))
+                )
                 return flask.jsonify(response)
         time.sleep(0.1)
-    logger.debug('No queries available for handler from worker {}', os.getpid())
+    logger.debug("No queries available for handler from worker {}", os.getpid())
     return flask.jsonify(ListQueriesResponse(requests=[]))
 
 
-@webapp.route('/handler/queries/<id>', methods=['PUT'])
+@webapp.route("/handler/queries/<id>", methods=["PUT"])
 @basic_auth.login_required
 def put_query_result(id: str) -> Tuple[str, int]:
     """Fulfills an outstanding query.
 
     See PutQueryRequest for request body schema.
     """
-    logger.debug('Handler instructed to fulfill request {} from worker {}', id, os.getpid())
+    logger.debug(
+        "Handler instructed to fulfill request {} from worker {}", id, os.getpid()
+    )
     try:
-        request: PutQueryRequest = ImplicitDict.parse(flask.request.json, PutQueryRequest)
+        request: PutQueryRequest = ImplicitDict.parse(
+            flask.request.json, PutQueryRequest
+        )
     except ValueError as e:
-        msg = f'Could not parse PutQueryRequest due to {type(e).__name__} on worker {os.getpid()}: {str(e)}'
+        msg = f"Could not parse PutQueryRequest due to {type(e).__name__} on worker {os.getpid()}: {str(e)}"
         logger.error(msg)
-        return flask.jsonify({'message': msg}), 400
+        return flask.jsonify({"message": msg}), 400
     with db as tx:
         if id not in tx.queries:
-            msg = f'No outstanding request with ID {id} exists on worker {os.getpid()}'
+            msg = f"No outstanding request with ID {id} exists on worker {os.getpid()}"
             logger.error(msg)
-            return flask.jsonify({'message': msg}), 400
+            return flask.jsonify({"message": msg}), 400
         query: Query = tx.queries[id]
-        logger.debug('{} query {} handled with code {}', query.type, id, request.return_code)
+        logger.debug(
+            "{} query {} handled with code {}", query.type, id, request.return_code
+        )
         query.return_code = request.return_code
         query.response = request.response
         query.state = QueryState.Complete
-    logger.debug('Handler fulfilled request {} from worker {}', id, os.getpid())
-    return '', 204
+    logger.debug("Handler fulfilled request {} from worker {}", id, os.getpid())
+    return "", 204