feat: monitoring for endpoint errors (#24617)

posthog/api/monitoring.py

@@ -17,6 +17,12 @@ class Feature(StrEnum):
     labelnames=["endpoint", "method"],
 )
 
+API_REQUESTS_ERROR_COUNTER = Counter(
+    "api_requests_error",
+    "Number of errored API requests",
+    labelnames=["endpoint", "method"],
+)
+
 
 def monitor(*, feature: Feature | None, endpoint: str, method: str) -> Callable:
     """
@@ -30,8 +36,11 @@ def wrapper(*args, **kwargs):
 
             if feature:
                 set_tag("feature", feature.value)
-
-            return func(*args, **kwargs)
+            try:
+                return func(*args, **kwargs)
+            except Exception:
+                API_REQUESTS_ERROR_COUNTER.labels(endpoint=endpoint, method=method).inc()
+                raise
 
         return wrapper
 

posthog/clickhouse/client/execute.py

@@ -23,7 +23,7 @@
 QUERY_ERROR_COUNTER = Counter(
     "clickhouse_query_failure",
     "Query execution failure signal is dispatched when a query fails.",
-    labelnames=["exception_type"],
+    labelnames=["exception_type", "query_type"],
 )
 
 QUERY_EXECUTION_TIME_GAUGE = Gauge(
@@ -146,7 +146,9 @@ def sync_execute(
             )
         except Exception as e:
             err = wrap_query_error(e)
-            QUERY_ERROR_COUNTER.labels(exception_type=type(err).__name__).inc()
+            exception_type = type(err).__name__
+            set_tag("clickhouse_exception_type", exception_type)
+            QUERY_ERROR_COUNTER.labels(exception_type=exception_type, query_type=query_type).inc()
 
             raise err from e
         finally: