From a1b6ae5203c7276f8ee04e0a1d577c998c4bca85 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 4 Oct 2023 13:44:08 +0100 Subject: [PATCH] fix: small console log fixes (#17768) * clearer ts type usage * case insensitive search --- .../services/console-logs-ingester.ts | 4 +- .../src/worker/ingestion/process-event.ts | 8 ++- ...sion_recording_list_from_replay_summary.py | 5 +- ...on_recording_list_from_session_replay.ambr | 65 ++++++++++++++++--- ...sion_recording_list_from_session_replay.py | 15 +++++ 5 files changed, 82 insertions(+), 15 deletions(-) diff --git a/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts b/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts index 1d9552b3f5068..4234637b8b68b 100644 --- a/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts +++ b/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts @@ -9,7 +9,7 @@ import { retryOnDependencyUnavailableError } from '../../../../kafka/error-handl import { createKafkaProducer, disconnectProducer, flushProducer, produce } from '../../../../kafka/producer' import { PluginsServerConfig } from '../../../../types' import { status } from '../../../../utils/status' -import { gatherConsoleLogEvents } from '../../../../worker/ingestion/process-event' +import { ConsoleLogEntry, gatherConsoleLogEvents } from '../../../../worker/ingestion/process-event' import { eventDroppedCounter } from '../../metrics' import { IncomingRecordingMessage } from '../types' import { OffsetHighWaterMarker } from './offset-high-water-marker' @@ -128,7 +128,7 @@ export class ConsoleLogsIngester { consoleLogEventsCounter.inc(consoleLogEvents.length) - return consoleLogEvents.map((cle) => + return consoleLogEvents.map((cle: ConsoleLogEntry) => produce({ producer, topic: KAFKA_LOG_ENTRIES, diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 21b4995a2a6d8..ce8d052e95ca8 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -287,7 +287,7 @@ export interface SummarizedSessionRecordingEvent { message_count: number } -type ConsoleLogEntry = { +export type ConsoleLogEntry = { team_id: number message: string log_level: 'info' | 'warn' | 'error' @@ -350,7 +350,11 @@ enum RRWebEventSource { StyleDeclaration = 1, Selection = 1, } -export const gatherConsoleLogEvents = (team_id: number, session_id: string, events: RRWebEvent[]) => { +export const gatherConsoleLogEvents = ( + team_id: number, + session_id: string, + events: RRWebEvent[] +): ConsoleLogEntry[] => { const consoleLogEntries: ConsoleLogEntry[] = [] events.forEach((event) => { diff --git a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py index 0ef6a674a113b..2588fbfaae0a8 100644 --- a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py +++ b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py @@ -57,7 +57,8 @@ def _get_filter_by_log_text_session_ids_clause( # we return this _even_ if there are no matching ids since if there are no matching ids # then no sessions can match... - return f'AND "{column_name}" in %(session_ids)s', {"session_ids": [x[0] for x in matching_session_ids]} + # sorted so that snapshots are consistent + return f'AND "{column_name}" in %(session_ids)s', {"session_ids": sorted([x[0] for x in matching_session_ids])} def _get_filter_by_provided_session_ids_clause( @@ -108,7 +109,7 @@ def __init__( {events_timestamp_clause} WHERE 1=1 {console_log_clause} - AND position(%(console_search_query)s in message) > 0 + AND positionCaseInsensitive(message, %(console_search_query)s) > 0 """ @property diff --git a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr index 9505ae1f31cff..1091b39b7af83 100644 --- a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr +++ b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr @@ -2656,7 +2656,7 @@ AND timestamp <= '2021-01-22 08:00:00' WHERE 1=1 AND level in ['warn', 'error'] - AND position('message 4' in message) > 0 + AND positionCaseInsensitive(message, 'message 4') > 0 ' --- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.1 @@ -2682,7 +2682,7 @@ AND s.min_first_timestamp >= '2020-12-31 20:00:00' AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in ['with-two-session-0b02c31f-341b-49cb-93d2-e8a2e4bf8330', 'with-errors-session-ea3f51a1-816a-4de4-9770-93f09d087477', 'with-warns-session-365c77cc-7d10-4925-816c-6f4930d75728'] + AND "session_id" in ['with-errors-session-fb398196-a87f-4f98-91eb-e7dccad4efcf', 'with-two-session-a4cc7feb-93e3-48c5-9daa-0e95538c5bb4', 'with-warns-session-b90d7779-7f4d-409c-bf01-e45e2c621b83'] GROUP BY session_id HAVING 1=1 AND (console_warn_count > 0 @@ -2703,7 +2703,7 @@ AND timestamp <= '2021-01-22 08:00:00' WHERE 1=1 AND level in ['warn', 'error'] - AND position('message 5' in message) > 0 + AND positionCaseInsensitive(message, 'message 5') > 0 ' --- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.3 @@ -2729,7 +2729,7 @@ AND s.min_first_timestamp >= '2020-12-31 20:00:00' AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in ['with-warns-session-365c77cc-7d10-4925-816c-6f4930d75728'] + AND "session_id" in ['with-warns-session-b90d7779-7f4d-409c-bf01-e45e2c621b83'] GROUP BY session_id HAVING 1=1 AND (console_warn_count > 0 @@ -2749,8 +2749,8 @@ AND timestamp >= '2021-01-13 12:00:00' AND timestamp <= '2021-01-22 08:00:00' WHERE 1=1 - AND level in ['log'] - AND position('message 5' in message) > 0 + AND level in ['warn', 'error'] + AND positionCaseInsensitive(message, 'MESSAGE 5') > 0 ' --- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.5 @@ -2776,10 +2776,11 @@ AND s.min_first_timestamp >= '2020-12-31 20:00:00' AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.max_last_timestamp <= '2021-01-21 20:00:00' - AND "session_id" in [] + AND "session_id" in ['with-warns-session-b90d7779-7f4d-409c-bf01-e45e2c621b83'] GROUP BY session_id HAVING 1=1 - AND (console_log_count > 0) + AND (console_warn_count > 0 + OR console_error_count > 0) ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2796,7 +2797,7 @@ AND timestamp <= '2021-01-22 08:00:00' WHERE 1=1 AND level in ['log'] - AND position('message 5' in message) > 0 + AND positionCaseInsensitive(message, 'message 5') > 0 ' --- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.7 @@ -2831,6 +2832,52 @@ OFFSET 0 ' --- +# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.8 + ' + + SELECT distinct log_source_id as session_id + FROM log_entries PREWHERE team_id = 2 + AND timestamp >= '2020-12-31 20:00:00' + AND timestamp <= now() + AND timestamp >= '2021-01-13 12:00:00' + AND timestamp <= '2021-01-22 08:00:00' + WHERE 1=1 + AND level in ['log'] + AND positionCaseInsensitive(message, 'message 5') > 0 + ' +--- +# name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_by_console_text.9 + ' + + SELECT s.session_id, + any(s.team_id), + any(s.distinct_id), + min(s.min_first_timestamp) as start_time, + max(s.max_last_timestamp) as end_time, + dateDiff('SECOND', start_time, end_time) as duration, + argMinMerge(s.first_url) as first_url, + sum(s.click_count), + sum(s.keypress_count), + sum(s.mouse_activity_count), + sum(s.active_milliseconds)/1000 as active_seconds, + duration-active_seconds as inactive_seconds, + sum(s.console_log_count) as console_log_count, + sum(s.console_warn_count) as console_warn_count, + sum(s.console_error_count) as console_error_count + FROM session_replay_events s + WHERE s.team_id = 2 + AND s.min_first_timestamp >= '2020-12-31 20:00:00' + AND s.min_first_timestamp >= '2021-01-14 00:00:00' + AND s.max_last_timestamp <= '2021-01-21 20:00:00' + AND "session_id" in [] + GROUP BY session_id + HAVING 1=1 + AND (console_log_count > 0) + ORDER BY start_time DESC + LIMIT 51 + OFFSET 0 + ' +--- # name: TestClickhouseSessionRecordingsListFromSessionReplay.test_filter_for_recordings_with_console_errors ' diff --git a/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py index 2b9c182576f89..6abcd8d9389b8 100644 --- a/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py +++ b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py @@ -2054,6 +2054,21 @@ def test_filter_for_recordings_by_console_text(self): ] ) + filter = SessionRecordingsFilter( + team=self.team, + # match is case-insensitive + data={"console_logs": ["warn", "error"], "console_search_query": "MESSAGE 5"}, + ) + + session_recording_list_instance = SessionRecordingListFromReplaySummary(filter=filter, team=self.team) + (session_recordings, _) = session_recording_list_instance.run() + + assert sorted([sr["session_id"] for sr in session_recordings]) == sorted( + [ + with_warns_session_id, + ] + ) + filter = SessionRecordingsFilter( team=self.team, # message 5 does not match log level "log