From c31db01bae79c3e6ce8f1b1ca8e1e5129734ec25 Mon Sep 17 00:00:00 2001
From: James Greenhill <fuziontech@gmail.com>
Date: Tue, 1 Oct 2024 10:50:05 -0700
Subject: [PATCH] chore: continue on timeout for CH lightweight deletes
 (#25319)

---
 posthog/models/async_deletion/delete_events.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/posthog/models/async_deletion/delete_events.py b/posthog/models/async_deletion/delete_events.py
index 36ca66e03d697..0348be5f23c07 100644
--- a/posthog/models/async_deletion/delete_events.py
+++ b/posthog/models/async_deletion/delete_events.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+from clickhouse_driver.errors import SocketTimeoutError
 from prometheus_client import Counter
 
 from posthog.client import sync_execute
@@ -69,11 +70,18 @@ def process(self, deletions: list[AsyncDeletion]):
             # If the query size is greater than the max predicate size, execute the query and reset the query predicate
             if query_size > MAX_QUERY_SIZE:
                 logger.debug(f"Executing query with args: {args}")
-                sync_execute(
-                    query,
-                    args,
-                    settings={},
-                )
+                try:
+                    sync_execute(
+                        query,
+                        args,
+                        settings={},
+                    )
+                except SocketTimeoutError:
+                    # This is unfortunately needed because currently all lightweight deletes are executed sync
+                    logger.warning(
+                        "ClickHouse query timed out during async deletion. This is expected. Continuing with next batch.",
+                        exc_info=True,
+                    )
 
                 conditions, args = [], {}