From 390f6c70c22f0ef635e4a8a0600cb803d2e9ea92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Wed, 15 Nov 2023 14:54:10 +0100 Subject: [PATCH] fix: Read all logs is case insensitive (#18648) * fix: Read all logs is case insensitive * Update query snapshots * Update query snapshots --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .../api/test/batch_exports/test_log_entry.py | 48 +++++++++++++++++++ posthog/batch_exports/models.py | 4 +- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/posthog/api/test/batch_exports/test_log_entry.py b/posthog/api/test/batch_exports/test_log_entry.py index 23d59b68e924a..8012766464ffd 100644 --- a/posthog/api/test/batch_exports/test_log_entry.py +++ b/posthog/api/test/batch_exports/test_log_entry.py @@ -171,6 +171,54 @@ def test_log_level_filter(batch_export, team, level): assert results[1].batch_export_id == str(batch_export["id"]) +@pytest.mark.django_db +@pytest.mark.parametrize( + "level", + [ + BatchExportLogEntryLevel.INFO, + BatchExportLogEntryLevel.WARNING, + BatchExportLogEntryLevel.ERROR, + BatchExportLogEntryLevel.DEBUG, + ], +) +def test_log_level_filter_with_lowercase(batch_export, team, level): + """Test fetching a batch export log entries of a particular level.""" + with freeze_time("2023-09-22 01:00:00"): + for message in ("Test log 1", "Test log 2"): + create_batch_export_log_entry( + team_id=team.pk, + batch_export_id=str(batch_export["id"]), + run_id=None, + message=message, + level=level.lower(), + ) + + results = [] + timeout = 10 + start = dt.datetime.utcnow() + + while not results: + results = fetch_batch_export_log_entries( + team_id=team.pk, + batch_export_id=batch_export["id"], + level_filter=[level], + after=dt.datetime(2023, 9, 22, 0, 59, 59), + before=dt.datetime(2023, 9, 22, 1, 0, 1), + ) + if (dt.datetime.utcnow() - start) > dt.timedelta(seconds=timeout): + break + + results.sort(key=lambda record: record.message) + + assert len(results) == 2 + assert results[0].message == "Test log 1" + assert results[0].level == level + assert results[0].batch_export_id == str(batch_export["id"]) + assert results[1].message == "Test log 2" + assert results[1].level == level + assert results[1].batch_export_id == str(batch_export["id"]) + + @pytest.mark.django_db def test_batch_export_log_api(client, batch_export, team): """Test fetching batch export log entries using the API.""" diff --git a/posthog/batch_exports/models.py b/posthog/batch_exports/models.py index 30ad08bc13c86..ab1a7b8b80db0 100644 --- a/posthog/batch_exports/models.py +++ b/posthog/batch_exports/models.py @@ -242,11 +242,11 @@ def fetch_batch_export_log_entries( clickhouse_where_parts.append("message ILIKE %(search)s") clickhouse_kwargs["search"] = f"%{search}%" if len(level_filter) > 0: - clickhouse_where_parts.append("level in %(levels)s") + clickhouse_where_parts.append("upper(level) in %(levels)s") clickhouse_kwargs["levels"] = level_filter clickhouse_query = f""" - SELECT team_id, log_source_id AS batch_export_id, instance_id AS run_id, timestamp, level, message FROM log_entries + SELECT team_id, log_source_id AS batch_export_id, instance_id AS run_id, timestamp, upper(level) as level, message FROM log_entries WHERE {' AND '.join(clickhouse_where_parts)} ORDER BY timestamp DESC {f'LIMIT {limit}' if limit else ''} """