WIP

apitally/client/request_logging.py

@@ -39,9 +39,10 @@ class RequestLoggingConfig:
 
 class RequestDict(TypedDict):
     method: str
-    path: str
+    path: Optional[str]
     url: str
     headers: Dict[str, str]
+    consumer: Optional[str]
 
 
 class ResponseDict(TypedDict):

apitally/django.py

@@ -16,6 +16,7 @@
 from apitally.client.client_threading import ApitallyClient
 from apitally.client.consumers import Consumer as ApitallyConsumer
 from apitally.client.logging import get_logger
+from apitally.client.request_logging import RequestLoggingConfig
 from apitally.common import get_versions
 
 
@@ -24,14 +25,15 @@
     from ninja import NinjaAPI
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]
 logger = get_logger(__name__)
 
 
 @dataclass
 class ApitallyMiddlewareConfig:
     client_id: str
     env: str
+    request_logging_config: Optional[RequestLoggingConfig]
     app_version: Optional[str]
     identify_consumer_callback: Optional[Callable[[HttpRequest], Union[str, ApitallyConsumer, None]]]
     urlconfs: List[Optional[str]]
@@ -75,13 +77,15 @@ def configure(
         cls,
         client_id: str,
         env: str = "dev",
+        request_logging_config: Optional[RequestLoggingConfig] = None,
         app_version: Optional[str] = None,
         identify_consumer_callback: Optional[str] = None,
         urlconf: Optional[Union[List[Optional[str]], str]] = None,
     ) -> None:
         cls.config = ApitallyMiddlewareConfig(
             client_id=client_id,
             env=env,
+            request_logging_config=request_logging_config,
             app_version=app_version,
             identify_consumer_callback=import_string(identify_consumer_callback)
             if identify_consumer_callback
@@ -93,15 +97,22 @@ def __call__(self, request: HttpRequest) -> HttpResponse:
         start_time = time.perf_counter()
         response = self.get_response(request)
         response_time = time.perf_counter() - start_time
+        response_size = (
+            _to_int(response["Content-Length"])
+            if response.has_header("Content-Length")
+            else (len(response.content) if not response.streaming else None)
+        )
         path = self.get_path(request)
+
+        try:
+            consumer = self.get_consumer(request)
+            consumer_identifier = consumer.identifier if consumer else None
+            self.client.consumer_registry.add_or_update_consumer(consumer)
+        except Exception:  # pragma: no cover
+            logger.exception("Failed to get consumer for request")
+            consumer_identifier = None
+
         if request.method is not None and request.method != "OPTIONS" and path is not None:
-            try:
-                consumer = self.get_consumer(request)
-                consumer_identifier = consumer.identifier if consumer else None
-                self.client.consumer_registry.add_or_update_consumer(consumer)
-            except Exception:  # pragma: no cover
-                logger.exception("Failed to get consumer for request")
-                consumer_identifier = None
             try:
                 self.client.request_counter.add_request(
                     consumer=consumer_identifier,
@@ -110,12 +121,11 @@ def __call__(self, request: HttpRequest) -> HttpResponse:
                     status_code=response.status_code,
                     response_time=response_time,
                     request_size=request.headers.get("Content-Length"),
-                    response_size=response["Content-Length"]
-                    if response.has_header("Content-Length")
-                    else (len(response.content) if not response.streaming else None),
+                    response_size=response_size,
                 )
             except Exception:  # pragma: no cover
                 logger.exception("Failed to log request metadata")
+
             if (
                 response.status_code == 422
                 and (content_type := response.get("Content-Type")) is not None
@@ -134,6 +144,7 @@ def __call__(self, request: HttpRequest) -> HttpResponse:
                             )
                 except Exception:  # pragma: no cover
                     logger.exception("Failed to log validation errors")
+
             if response.status_code == 500 and hasattr(request, "unhandled_exception"):
                 try:
                     self.client.server_error_counter.add_server_error(
@@ -144,6 +155,24 @@ def __call__(self, request: HttpRequest) -> HttpResponse:
                     )
                 except Exception:  # pragma: no cover
                     logger.exception("Failed to log server error")
+
+        if self.client.request_logger.enabled and request.method is not None:
+            self.client.request_logger.log_request(
+                request={
+                    "method": request.method,
+                    "path": path,
+                    "url": request.build_absolute_uri(),
+                    "headers": dict(request.headers),
+                    "consumer": consumer_identifier,
+                },
+                response={
+                    "status_code": response.status_code,
+                    "response_time": response_time,
+                    "headers": dict(response.items()),
+                    "size": response_size,
+                },
+            )
+
         return response
 
     def process_exception(self, request: HttpRequest, exception: Exception) -> None:
@@ -335,3 +364,12 @@ def _check_import(name: str) -> bool:
         return True
     except ImportError:
         return False
+
+
+def _to_int(x: Union[str, int, None]) -> Optional[int]:
+    if x is None:
+        return None
+    try:
+        return int(x)
+    except ValueError:
+        return None

apitally/django_ninja.py

@@ -1,4 +1,4 @@
-from apitally.django import ApitallyConsumer, ApitallyMiddleware
+from apitally.django import ApitallyConsumer, ApitallyMiddleware, RequestLoggingConfig
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]

apitally/django_rest_framework.py

@@ -1,4 +1,4 @@
-from apitally.django import ApitallyConsumer, ApitallyMiddleware
+from apitally.django import ApitallyConsumer, ApitallyMiddleware, RequestLoggingConfig
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]

apitally/fastapi.py

@@ -1,4 +1,4 @@
-from apitally.starlette import ApitallyConsumer, ApitallyMiddleware
+from apitally.starlette import ApitallyConsumer, ApitallyMiddleware, RequestLoggingConfig
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]

apitally/flask.py

@@ -6,13 +6,14 @@
 from warnings import warn
 
 from flask import Flask, g
-from flask.wrappers import Response
+from flask.wrappers import Request, Response
 from werkzeug.datastructures import Headers
 from werkzeug.exceptions import NotFound
 from werkzeug.test import Client
 
 from apitally.client.client_threading import ApitallyClient
 from apitally.client.consumers import Consumer as ApitallyConsumer
+from apitally.client.request_logging import RequestLoggingConfig
 from apitally.common import get_versions
 
 
@@ -21,7 +22,7 @@
     from werkzeug.routing.map import Map
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]
 
 
 class ApitallyMiddleware:
@@ -30,13 +31,14 @@ def __init__(
         app: Flask,
         client_id: str,
         env: str = "dev",
+        request_logging_config: Optional[RequestLoggingConfig] = None,
         app_version: Optional[str] = None,
         openapi_url: Optional[str] = None,
     ) -> None:
         self.app = app
         self.wsgi_app = app.wsgi_app
         self.patch_handle_exception()
-        self.client = ApitallyClient(client_id=client_id, env=env)
+        self.client = ApitallyClient(client_id=client_id, env=env, request_logging_config=request_logging_config)
         self.client.start_sync_loop()
         self.delayed_set_startup_data(app_version, openapi_url)
 
@@ -90,36 +92,56 @@ def add_request(
         response_time: float,
         response_headers: Headers,
     ) -> None:
-        rule, is_handled_path = self.get_rule(environ)
-        if is_handled_path and environ["REQUEST_METHOD"] != "OPTIONS":
+        path = self.get_path(environ)
+        request = Request(environ, populate_request=False, shallow=True)
+        response_size = response_headers.get("Content-Length", type=int)
+
+        if path is not None and request.method != "OPTIONS":
             consumer = self.get_consumer()
             consumer_identifier = consumer.identifier if consumer else None
             self.client.consumer_registry.add_or_update_consumer(consumer)
             self.client.request_counter.add_request(
                 consumer=consumer_identifier,
-                method=environ["REQUEST_METHOD"],
-                path=rule,
+                method=request.method,
+                path=path,
                 status_code=status_code,
                 response_time=response_time,
-                request_size=environ.get("CONTENT_LENGTH"),
-                response_size=response_headers.get("Content-Length", type=int),
+                request_size=request.content_length,
+                response_size=response_size,
             )
             if status_code == 500 and "unhandled_exception" in g:
                 self.client.server_error_counter.add_server_error(
                     consumer=consumer_identifier,
-                    method=environ["REQUEST_METHOD"],
-                    path=rule,
+                    method=request.method,
+                    path=path,
                     exception=g.unhandled_exception,
                 )
 
-    def get_rule(self, environ: WSGIEnvironment) -> Tuple[str, bool]:
+        if self.client.request_logger.enabled:
+            self.client.request_logger.log_request(
+                request={
+                    "method": request.method,
+                    "path": path,
+                    "url": request.url,
+                    "headers": dict(request.headers),
+                    "consumer": consumer_identifier,
+                },
+                response={
+                    "status_code": status_code,
+                    "response_time": response_time,
+                    "headers": dict(response_headers),
+                    "size": response_size,
+                },
+            )
+
+    def get_path(self, environ: WSGIEnvironment) -> Optional[str]:
         url_adapter = self.app.url_map.bind_to_environ(environ)
         try:
             endpoint, _ = url_adapter.match()
             rule = self.app.url_map._rules_by_endpoint[endpoint][0]
-            return rule.rule, True
+            return rule.rule
         except NotFound:
-            return environ["PATH_INFO"], False
+            return None
 
     def get_consumer(self) -> Optional[ApitallyConsumer]:
         if "apitally_consumer" in g and g.apitally_consumer: