From 8fd7b2385ae0ef470a574d479f82128bab53d132 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 22 Nov 2023 14:01:50 +0100 Subject: [PATCH] fix(plugin-server): don't sent app debug logs to CH --- plugin-server/src/types.ts | 7 +++--- plugin-server/src/utils/db/db.ts | 5 ++--- plugin-server/src/utils/db/utils.ts | 33 ++++++++++++----------------- posthog/api/capture.py | 5 +++-- posthog/api/utils.py | 6 ++++++ 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 700bf4f6cef895..af3d34f69cc18b 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -451,9 +451,10 @@ export enum PluginLogEntryType { export enum PluginLogLevel { Full = 0, // all logs - Debug = 1, // all except log - Warn = 2, // all except log and info - Critical = 3, // only error type and system source + Log = 1, // all except debug + Info = 2, // all expect log and debug + Warn = 3, // all except log, debug and info + Critical = 4, // only error type and system source } export interface PluginLogEntry { diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index cf64f86b600a29..f6999b3caae853 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -1076,10 +1076,9 @@ export class DB { public async queuePluginLogEntry(entry: LogEntryPayload): Promise { const { pluginConfig, source, message, type, timestamp, instanceId } = entry + const configuredLogLevel = pluginConfig.plugin?.log_level || PluginLogLevel.Log - const logLevel = pluginConfig.plugin?.log_level - - if (!shouldStoreLog(logLevel || PluginLogLevel.Full, source, type)) { + if (!shouldStoreLog(configuredLogLevel, type)) { return } diff --git a/plugin-server/src/utils/db/utils.ts b/plugin-server/src/utils/db/utils.ts index a4f940defdefb3..fbf9a1bea2018a 100644 --- a/plugin-server/src/utils/db/utils.ts +++ b/plugin-server/src/utils/db/utils.ts @@ -6,10 +6,10 @@ import { Counter } from 'prom-client' import { defaultConfig } from '../../config/config' import { KAFKA_PERSON } from '../../config/kafka-topics' -import { BasePerson, Person, RawPerson, TimestampFormat } from '../../types' +import { BasePerson, Person, PluginLogLevel, RawPerson, TimestampFormat } from '../../types' import { status } from '../../utils/status' import { castTimestampOrNow } from '../../utils/utils' -import { PluginLogEntrySource, PluginLogEntryType, PluginLogLevel } from './../../types' +import { PluginLogEntryType, PluginLogLevel } from './../../types' export function unparsePersonPartial(person: Partial): Partial { return { ...(person as BasePerson), ...(person.created_at ? { created_at: person.created_at.toISO() } : {}) } @@ -127,24 +127,19 @@ export function getFinalPostgresQuery(queryString: string, values: any[]): strin return queryString.replace(/\$([0-9]+)/g, (m, v) => JSON.stringify(values[parseInt(v) - 1])) } -export function shouldStoreLog( - pluginLogLevel: PluginLogLevel, - source: PluginLogEntrySource, - type: PluginLogEntryType -): boolean { - if (source === PluginLogEntrySource.System) { - return true +export function shouldStoreLog(pluginLogLevel: PluginLogLevel, type: PluginLogEntryType): boolean { + switch (pluginLogLevel) { + case PluginLogLevel.Full: + return true + case PluginLogLevel.Log: + return type !== PluginLogEntryType.Debug + case PluginLogLevel.Info: + return type !== PluginLogEntryType.Log && type !== PluginLogEntryType.Debug + case PluginLogLevel.Warn: + return type === PluginLogEntryType.Warn || type === PluginLogEntryType.Error + case PluginLogLevel.Critical: + return type === PluginLogEntryType.Error } - - if (pluginLogLevel === PluginLogLevel.Critical) { - return type === PluginLogEntryType.Error - } else if (pluginLogLevel === PluginLogLevel.Warn) { - return type !== PluginLogEntryType.Log && type !== PluginLogEntryType.Info - } else if (pluginLogLevel === PluginLogLevel.Debug) { - return type !== PluginLogEntryType.Log - } - - return true } // keep in sync with posthog/posthog/api/utils.py::safe_clickhouse_string diff --git a/posthog/api/capture.py b/posthog/api/capture.py index ac954a3b8d6d21..2bacc059a5aa1b 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -20,7 +20,7 @@ from statshog.defaults.django import statsd from token_bucket import Limiter, MemoryStorage -from posthog.api.utils import get_data, get_token, safe_clickhouse_string +from posthog.api.utils import get_data, get_token, safe_clickhouse_string, CAPTURE_KLUDGES_COUNTER from posthog.exceptions import generate_exception_response from posthog.kafka_client.client import ( KafkaProducer, @@ -75,7 +75,6 @@ labelnames=[LABEL_RESOURCE_TYPE, "token"], ) - PARTITION_KEY_CAPACITY_EXCEEDED_COUNTER = Counter( "capture_partition_key_capacity_exceeded_total", "Indicates that automatic partition override is active for a given key. Value incremented once a minute.", @@ -182,6 +181,7 @@ def _datetime_from_seconds_or_millis(timestamp: str) -> datetime: if len(timestamp) > 11: # assuming milliseconds / update "11" to "12" if year > 5138 (set a reminder!) timestamp_number = float(timestamp) / 1000 else: + CAPTURE_KLUDGES_COUNTER.labels(type="sent_at_seconds").inc() timestamp_number = int(timestamp) return datetime.fromtimestamp(timestamp_number, timezone.utc) @@ -201,6 +201,7 @@ def _get_sent_at(data, request) -> Tuple[Optional[datetime], Any]: if re.match(r"^\d+(?:\.\d+)?$", sent_at): return _datetime_from_seconds_or_millis(sent_at), None + CAPTURE_KLUDGES_COUNTER.labels(type="sent_at_not_numbers").inc() return parser.isoparse(sent_at), None except Exception as error: statsd.incr("capture_endpoint_invalid_sent_at") diff --git a/posthog/api/utils.py b/posthog/api/utils.py index 1953b86029c16f..96cb6c6dfd2503 100644 --- a/posthog/api/utils.py +++ b/posthog/api/utils.py @@ -26,6 +26,12 @@ logger = structlog.get_logger(__name__) +CAPTURE_KLUDGES_COUNTER = Counter( + "capture_kludges_total", + "Occurrences of fallbacks and kludges, per library, to check dead code we could cleanup.", + labelnames=["type"], +) + class PaginationMode(Enum): next = auto()