Merge branch 'main' into interactions_and_invalid_op_checks

monitoring/Dockerfile

@@ -9,7 +9,7 @@
 #
 # This image is intended to be built from the repository root context/folder.
 
-FROM python:3.8-slim
+FROM python:3.11-slim
 # Not -alpine because: https://stackoverflow.com/a/58028091/651139
 
 RUN apt-get update && apt-get install -y openssl curl libgeos-dev gcc && apt-get install ca-certificates

monitoring/mock_uss/database.py

@@ -25,9 +25,7 @@ def from_exception(trigger: str, e: BaseException):
             trigger=trigger,
             type=type(e).__name__,
             message=str(e),
-            stacktrace="".join(
-                traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)
-            ),
+            stacktrace="".join(traceback.format_exception(e)),
         )
 
 

monitoring/mock_uss/scd_injection/routes_injection.py

@@ -68,9 +68,7 @@
 
 
 def _make_stacktrace(e) -> str:
-    return "".join(
-        traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)
-    )
+    return "".join(traceback.format_exception(e))
 
 
 @webapp.route("/scdsc/v1/status", methods=["GET"])

monitoring/mock_uss/server.py

@@ -21,9 +21,7 @@
 
 
 def _get_trace(e: Exception) -> str:
-    return "".join(
-        traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)
-    )
+    return "".join(traceback.format_exception(e))
 
 
 class TaskTrigger(str, Enum):

monitoring/monitorlib/auth.py

@@ -174,10 +174,6 @@ def issue_token(self, intended_audience: str, scopes: List[str]) -> str:
             )
         return response.json()["access_token"]
 
-    def get_sub(self) -> Optional[str]:
-        """directly return the configured `sub` value"""
-        return self._sub
-
 
 class _SessionIssuer:
     """Helper for issuing tokens using a pre-configured Google session."""

monitoring/monitorlib/fetch/__init__.py

@@ -8,6 +8,7 @@
 from enum import Enum
 from urllib.parse import urlparse
 
+import aiohttp
 import flask
 from loguru import logger
 import requests
@@ -149,6 +150,20 @@ def describe_response(resp: requests.Response) -> ResponseDescription:
     return ResponseDescription(**kwargs)
 
 
+def describe_aiohttp_response(
+    status: int, headers: Dict, resp_json: Dict, duration: datetime.timedelta
+) -> ResponseDescription:
+    kwargs = {
+        "code": status,
+        "headers": headers,
+        "elapsed_s": duration.total_seconds(),
+        "reported": StringBasedDateTime(datetime.datetime.utcnow()),
+        "json": resp_json,
+    }
+
+    return ResponseDescription(**kwargs)
+
+
 def describe_flask_response(resp: flask.Response, elapsed_s: float):
     headers = {k: v for k, v in resp.headers.items()}
     kwargs = {
@@ -293,11 +308,7 @@ def __init__(self, msg: str, queries: Optional[Union[Query, List[Query]]] = None
 
     @property
     def stacktrace(self) -> str:
-        return "".join(
-            traceback.format_exception(
-                etype=QueryError, value=self, tb=self.__traceback__
-            )
-        )
+        return "".join(traceback.format_exception(self))
 
 
 yaml.add_representer(Query, Representer.represent_dict)

monitoring/monitorlib/infrastructure.py

@@ -1,6 +1,7 @@
 import asyncio
 import datetime
 import functools
+from enum import Enum
 from typing import Dict, List, Optional
 import urllib.parse
 from aiohttp import ClientSession, ClientResponse
@@ -39,6 +40,7 @@ def issue_token(self, intended_audience: str, scopes: List[str]) -> str:
     def get_headers(self, url: str, scopes: List[str] = None) -> Dict[str, str]:
         if scopes is None:
             scopes = ALL_SCOPES
+        scopes = [s.value if isinstance(s, Enum) else s for s in scopes]
         intended_audience = urllib.parse.urlparse(url).hostname
         scope_string = " ".join(scopes)
         if intended_audience not in self._tokens:

monitoring/monitorlib/multiprocessing.py

@@ -46,7 +46,7 @@ def __init__(
         """
         self._lock = multiprocessing.RLock()
         self._shared_memory = multiprocessing.shared_memory.SharedMemory(
-            create=True, size=capacity_bytes + self.SIZE_BYTES
+            create=True, size=int(capacity_bytes + self.SIZE_BYTES)
         )
         self._encoder = (
             encoder

monitoring/monitorlib/mutate/rid.py

@@ -3,6 +3,7 @@
 
 from implicitdict import ImplicitDict
 import s2sphere
+from uas_standards import Operation
 
 from monitoring.monitorlib.fetch.rid import RIDQuery, Subscription, ISA
 from monitoring.monitorlib.rid import RIDVersion
@@ -439,22 +440,19 @@ class ISAChange(ImplicitDict):
     """Mapping from USS base URL to change notification query"""
 
 
-def put_isa(
+def build_isa_request_body(
     area_vertices: List[s2sphere.LatLng],
     alt_lo: float,
     alt_hi: float,
     start_time: datetime.datetime,
     end_time: datetime.datetime,
     uss_base_url: str,
-    isa_id: str,
     rid_version: RIDVersion,
-    utm_client: infrastructure.UTMClientSession,
-    isa_version: Optional[str] = None,
-    participant_id: Optional[str] = None,
-) -> ISAChange:
-    mutation = "create" if isa_version is None else "update"
+) -> Dict[str, any]:
+    """Build the http request body expected to PUT or UPDATE an ISA on a DSS,
+    in accordance with the specified rid_version."""
     if rid_version == RIDVersion.f3411_19:
-        body = {
+        return {
             "extents": rid_v1.make_volume_4d(
                 area_vertices,
                 alt_lo,
@@ -465,25 +463,8 @@ def put_isa(
             "flights_url": uss_base_url
             + v19.api.OPERATIONS[v19.api.OperationID.SearchFlights].path,
         }
-        if isa_version is None:
-            op = v19.api.OPERATIONS[v19.api.OperationID.CreateIdentificationServiceArea]
-            url = op.path.format(id=isa_id)
-        else:
-            op = v19.api.OPERATIONS[v19.api.OperationID.UpdateIdentificationServiceArea]
-            url = op.path.format(id=isa_id, version=isa_version)
-        dss_response = ChangedISA(
-            mutation=mutation,
-            v19_query=fetch.query_and_describe(
-                utm_client,
-                op.verb,
-                url,
-                json=body,
-                scope=v19.constants.Scope.Write,
-                participant_id=participant_id,
-            ),
-        )
     elif rid_version == RIDVersion.f3411_22a:
-        body = {
+        return {
             "extents": rid_v2.make_volume_4d(
                 area_vertices,
                 alt_lo,
@@ -493,16 +474,79 @@ def put_isa(
             ),
             "uss_base_url": uss_base_url,
         }
+    else:
+        raise NotImplementedError(
+            f"Cannot build ISA payload for RID version {rid_version}"
+        )
+
+
+def build_isa_url(
+    rid_version: RIDVersion, isa_id: str, isa_version: Optional[str] = None
+) -> (Operation, str):
+    """Build the required URL to create, get, update or delete an ISA on a DSS,
+    in accordance with the specified rid_version and isa_version, if it is available.
+
+    Note that for mutations and deletions, isa_version must be provided.
+    """
+    if rid_version == RIDVersion.f3411_19:
+        if isa_version is None:
+            op = v19.api.OPERATIONS[v19.api.OperationID.CreateIdentificationServiceArea]
+            return (op, op.path.format(id=isa_id))
+        else:
+            op = v19.api.OPERATIONS[v19.api.OperationID.UpdateIdentificationServiceArea]
+            return (op, op.path.format(id=isa_id, version=isa_version))
+    elif rid_version == RIDVersion.f3411_22a:
         if isa_version is None:
             op = v22a.api.OPERATIONS[
                 v22a.api.OperationID.CreateIdentificationServiceArea
             ]
-            url = op.path.format(id=isa_id)
+            return (op, op.path.format(id=isa_id))
         else:
             op = v22a.api.OPERATIONS[
                 v22a.api.OperationID.UpdateIdentificationServiceArea
             ]
-            url = op.path.format(id=isa_id, version=isa_version)
+            return (op, op.path.format(id=isa_id, version=isa_version))
+    else:
+        raise NotImplementedError(f"Cannot build ISA URL for RID version {rid_version}")
+
+
+def put_isa(
+    area_vertices: List[s2sphere.LatLng],
+    alt_lo: float,
+    alt_hi: float,
+    start_time: datetime.datetime,
+    end_time: datetime.datetime,
+    uss_base_url: str,
+    isa_id: str,
+    rid_version: RIDVersion,
+    utm_client: infrastructure.UTMClientSession,
+    isa_version: Optional[str] = None,
+    participant_id: Optional[str] = None,
+) -> ISAChange:
+    mutation = "create" if isa_version is None else "update"
+    body = build_isa_request_body(
+        area_vertices,
+        alt_lo,
+        alt_hi,
+        start_time,
+        end_time,
+        uss_base_url,
+        rid_version,
+    )
+    (op, url) = build_isa_url(rid_version, isa_id, isa_version)
+    if rid_version == RIDVersion.f3411_19:
+        dss_response = ChangedISA(
+            mutation=mutation,
+            v19_query=fetch.query_and_describe(
+                utm_client,
+                op.verb,
+                url,
+                json=body,
+                scope=v19.constants.Scope.Write,
+                participant_id=participant_id,
+            ),
+        )
+    elif rid_version == RIDVersion.f3411_22a:
         dss_response = ChangedISA(
             mutation=mutation,
             v22a_query=fetch.query_and_describe(

monitoring/monitorlib/rid.py

@@ -57,6 +57,15 @@ def openapi_search_isas_response_path(self) -> str:
         else:
             raise ValueError(f"Unsupported RID version '{self}'")
 
+    @property
+    def openapi_get_isa_response_path(self) -> str:
+        if self == RIDVersion.f3411_19:
+            return schema_validation.F3411_19.GetIdentificationServiceAreaResponse
+        elif self == RIDVersion.f3411_22a:
+            return schema_validation.F3411_22a.GetIdentificationServiceAreaResponse
+        else:
+            raise ValueError(f"Unsupported RID version '{self}'")
+
     @property
     def openapi_put_isa_response_path(self) -> str:
         if self == RIDVersion.f3411_19:

monitoring/monitorlib/schema_validation.py

@@ -19,6 +19,9 @@ class F3411_19(str, Enum):
     SearchIdentificationServiceAreasResponse = (
         "components.schemas.SearchIdentificationServiceAreasResponse"
     )
+    GetIdentificationServiceAreaResponse = (
+        "components.schemas.GetIdentificationServiceAreaResponse"
+    )
     PutIdentificationServiceAreaResponse = (
         "components.schemas.PutIdentificationServiceAreaResponse"
     )
@@ -34,6 +37,9 @@ class F3411_22a(str, Enum):
     SearchIdentificationServiceAreasResponse = (
         "components.schemas.SearchIdentificationServiceAreasResponse"
     )
+    GetIdentificationServiceAreaResponse = (
+        "components.schemas.GetIdentificationServiceAreaResponse"
+    )
     PutIdentificationServiceAreaResponse = (
         "components.schemas.PutIdentificationServiceAreaResponse"
     )

monitoring/prober/infrastructure.py

@@ -100,7 +100,7 @@ def wrapper_default_scope(*args, **kwargs):
 resource_type_code_descriptions: Dict[ResourceType, str] = {}
 
 
-# Next code: 374
+# Next code: 375
 def register_resource_type(code: int, description: str) -> ResourceType:
     """Register that the specified code refers to the described resource.
 

monitoring/prober/scd/test_constraints_with_subscriptions.py

@@ -67,7 +67,7 @@ def _make_sub_req(base_url: str, notify_ops: bool, notify_constraints: bool) ->
 
 def _read_both_scope(scd_api: str) -> str:
     if scd_api == scd.API_0_3_17:
-        return "{} {}".format(SCOPE_SC, SCOPE_CP)
+        return "{} {}".format(SCOPE_SC.value, SCOPE_CP.value)
     else:
         raise NotImplementedError("Unsupported API version {}".format(scd_api))
 

monitoring/uss_qualifier/reports/report.py

@@ -202,9 +202,7 @@ def create_from_exception(e: Exception):
             type=str(inspection.fullname(e.__class__)),
             message=str(e),
             timestamp=StringBasedDateTime(datetime.utcnow()),
-            stacktrace="".join(
-                traceback.format_exception(type(e), value=e, tb=e.__traceback__)
-            ),
+            stacktrace="".join(traceback.format_exception(e)),
         )
 
 

monitoring/uss_qualifier/resources/communications/client_identity.py

@@ -53,10 +53,10 @@ def subscriber(self) -> str:
             # sub might be none because no authentication has happened yet:
             # we force one using the client identify audience and scopes
 
-            # Do an initial token request so that adapter.get_sub() will return something
-            token = self._adapter.issue_token(
-                intended_audience=self.specification.whoami_audience,
-                scopes=[self.specification.whoami_scope],
+            # Trigger a caching initial token request so that adapter.get_sub() will return something
+            headers = self._adapter.get_headers(
+                f"https://{self.specification.whoami_audience}",
+                [self.specification.whoami_scope],
             )
 
             sub = self._adapter.get_sub()
@@ -65,7 +65,7 @@ def subscriber(self) -> str:
                 raise ValueError(
                     f"subscriber is None, meaning `sub` claim was not found in payload of token, "
                     f"using {type(self._adapter).__name__} requesting {self.specification.whoami_scope} scope "
-                    f"for {self.specification.whoami_audience} audience: {token}"
+                    f"for {self.specification.whoami_audience} audience: {headers['Authorization'][len('Bearer: '):]}"
                 )
 
         return sub