From 601fd3a1ce8745d8746f726904632e22b15b30af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Fri, 14 Jun 2024 17:30:19 +0200 Subject: [PATCH] chore: Bump max memory usage for batch exports (#22895) * chore: Bump max memory usage for batch exports * fix: Set limt Co-authored-by: timgl --------- Co-authored-by: timgl --- posthog/settings/temporal.py | 1 + posthog/temporal/common/clickhouse.py | 1 + 2 files changed, 2 insertions(+) diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 75e5b9c3ea93b..140fa36dd44f9 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -27,6 +27,7 @@ ) CLICKHOUSE_MAX_EXECUTION_TIME: int = get_from_env("CLICKHOUSE_MAX_EXECUTION_TIME", 0, type_cast=int) +CLICKHOUSE_MAX_MEMORY_USAGE: int = get_from_env("CLICKHOUSE_MAX_MEMORY_USAGE", 100 * 1000 * 1000 * 1000, type_cast=int) CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT: int = get_from_env("CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT", 10000, type_cast=int) # Comma separated list of overrides in the format "team_id:block_size" CLICKHOUSE_MAX_BLOCK_SIZE_OVERRIDES: dict[int, int] = dict( diff --git a/posthog/temporal/common/clickhouse.py b/posthog/temporal/common/clickhouse.py index 2640bf95c1f97..ad8bfe8173e82 100644 --- a/posthog/temporal/common/clickhouse.py +++ b/posthog/temporal/common/clickhouse.py @@ -417,6 +417,7 @@ async def get_client( password=settings.CLICKHOUSE_PASSWORD, database=settings.CLICKHOUSE_DATABASE, max_execution_time=settings.CLICKHOUSE_MAX_EXECUTION_TIME, + max_memory_usage=settings.CLICKHOUSE_MAX_MEMORY_USAGE, max_block_size=max_block_size, output_format_arrow_string_as_string="true", **kwargs,