fix(plugin-server): don't sent app debug logs to CH

PostHog · Nov 22, 2023 · 8fd7b23 · 8fd7b23
1 parent 266c3ff
commit 8fd7b23
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 27 deletions.
diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts
@@ -451,9 +451,10 @@ export enum PluginLogEntryType {
 
 export enum PluginLogLevel {
     Full = 0, // all logs
-    Debug = 1, // all except log
-    Warn = 2, // all except log and info
-    Critical = 3, // only error type and system source
+    Log = 1, // all except debug
+    Info = 2, // all expect log and debug
+    Warn = 3, // all except log, debug and info
+    Critical = 4, // only error type and system source
 }
 
 export interface PluginLogEntry {

diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts
@@ -1076,10 +1076,9 @@ export class DB {
 
     public async queuePluginLogEntry(entry: LogEntryPayload): Promise<void> {
         const { pluginConfig, source, message, type, timestamp, instanceId } = entry
+        const configuredLogLevel = pluginConfig.plugin?.log_level || PluginLogLevel.Log
 
-        const logLevel = pluginConfig.plugin?.log_level
-
-        if (!shouldStoreLog(logLevel || PluginLogLevel.Full, source, type)) {
+        if (!shouldStoreLog(configuredLogLevel, type)) {
             return
         }
 

diff --git a/plugin-server/src/utils/db/utils.ts b/plugin-server/src/utils/db/utils.ts
@@ -6,10 +6,10 @@ import { Counter } from 'prom-client'
 
 import { defaultConfig } from '../../config/config'
 import { KAFKA_PERSON } from '../../config/kafka-topics'
-import { BasePerson, Person, RawPerson, TimestampFormat } from '../../types'
+import { BasePerson, Person, PluginLogLevel, RawPerson, TimestampFormat } from '../../types'
 import { status } from '../../utils/status'
 import { castTimestampOrNow } from '../../utils/utils'
-import { PluginLogEntrySource, PluginLogEntryType, PluginLogLevel } from './../../types'
+import { PluginLogEntryType, PluginLogLevel } from './../../types'
 
 export function unparsePersonPartial(person: Partial<Person>): Partial<RawPerson> {
     return { ...(person as BasePerson), ...(person.created_at ? { created_at: person.created_at.toISO() } : {}) }
@@ -127,24 +127,19 @@ export function getFinalPostgresQuery(queryString: string, values: any[]): strin
     return queryString.replace(/\$([0-9]+)/g, (m, v) => JSON.stringify(values[parseInt(v) - 1]))
 }
 
-export function shouldStoreLog(
-    pluginLogLevel: PluginLogLevel,
-    source: PluginLogEntrySource,
-    type: PluginLogEntryType
-): boolean {
-    if (source === PluginLogEntrySource.System) {
-        return true
+export function shouldStoreLog(pluginLogLevel: PluginLogLevel, type: PluginLogEntryType): boolean {
+    switch (pluginLogLevel) {
+        case PluginLogLevel.Full:
+            return true
+        case PluginLogLevel.Log:
+            return type !== PluginLogEntryType.Debug
+        case PluginLogLevel.Info:
+            return type !== PluginLogEntryType.Log && type !== PluginLogEntryType.Debug
+        case PluginLogLevel.Warn:
+            return type === PluginLogEntryType.Warn || type === PluginLogEntryType.Error
+        case PluginLogLevel.Critical:
+            return type === PluginLogEntryType.Error
     }
-
-    if (pluginLogLevel === PluginLogLevel.Critical) {
-        return type === PluginLogEntryType.Error
-    } else if (pluginLogLevel === PluginLogLevel.Warn) {
-        return type !== PluginLogEntryType.Log && type !== PluginLogEntryType.Info
-    } else if (pluginLogLevel === PluginLogLevel.Debug) {
-        return type !== PluginLogEntryType.Log
-    }
-
-    return true
 }
 
 // keep in sync with posthog/posthog/api/utils.py::safe_clickhouse_string

diff --git a/posthog/api/capture.py b/posthog/api/capture.py
@@ -20,7 +20,7 @@
 from statshog.defaults.django import statsd
 from token_bucket import Limiter, MemoryStorage
 
-from posthog.api.utils import get_data, get_token, safe_clickhouse_string
+from posthog.api.utils import get_data, get_token, safe_clickhouse_string, CAPTURE_KLUDGES_COUNTER
 from posthog.exceptions import generate_exception_response
 from posthog.kafka_client.client import (
     KafkaProducer,
@@ -75,7 +75,6 @@
     labelnames=[LABEL_RESOURCE_TYPE, "token"],
 )
 
-
 PARTITION_KEY_CAPACITY_EXCEEDED_COUNTER = Counter(
     "capture_partition_key_capacity_exceeded_total",
     "Indicates that automatic partition override is active for a given key. Value incremented once a minute.",
@@ -182,6 +181,7 @@ def _datetime_from_seconds_or_millis(timestamp: str) -> datetime:
     if len(timestamp) > 11:  # assuming milliseconds / update "11" to "12" if year > 5138 (set a reminder!)
         timestamp_number = float(timestamp) / 1000
     else:
+        CAPTURE_KLUDGES_COUNTER.labels(type="sent_at_seconds").inc()
         timestamp_number = int(timestamp)
 
     return datetime.fromtimestamp(timestamp_number, timezone.utc)
@@ -201,6 +201,7 @@ def _get_sent_at(data, request) -> Tuple[Optional[datetime], Any]:
         if re.match(r"^\d+(?:\.\d+)?$", sent_at):
             return _datetime_from_seconds_or_millis(sent_at), None
 
+        CAPTURE_KLUDGES_COUNTER.labels(type="sent_at_not_numbers").inc()
         return parser.isoparse(sent_at), None
     except Exception as error:
         statsd.incr("capture_endpoint_invalid_sent_at")

diff --git a/posthog/api/utils.py b/posthog/api/utils.py
@@ -26,6 +26,12 @@
 
 logger = structlog.get_logger(__name__)
 
+CAPTURE_KLUDGES_COUNTER = Counter(
+    "capture_kludges_total",
+    "Occurrences of fallbacks and kludges, per library, to check dead code we could cleanup.",
+    labelnames=["type"],
+)
+
 
 class PaginationMode(Enum):
     next = auto()