chore: Bump max memory usage for batch exports (#22895)

* chore: Bump max memory usage for batch exports * fix: Set limt Co-authored-by: timgl <[email protected]> --------- Co-authored-by: timgl <[email protected]>
PostHog · Jun 14, 2024 · 601fd3a · 601fd3a
1 parent fa00f35
commit 601fd3a
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
@@ -27,6 +27,7 @@
 )
 
 CLICKHOUSE_MAX_EXECUTION_TIME: int = get_from_env("CLICKHOUSE_MAX_EXECUTION_TIME", 0, type_cast=int)
+CLICKHOUSE_MAX_MEMORY_USAGE: int = get_from_env("CLICKHOUSE_MAX_MEMORY_USAGE", 100 * 1000 * 1000 * 1000, type_cast=int)
 CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT: int = get_from_env("CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT", 10000, type_cast=int)
 # Comma separated list of overrides in the format "team_id:block_size"
 CLICKHOUSE_MAX_BLOCK_SIZE_OVERRIDES: dict[int, int] = dict(

diff --git a/posthog/temporal/common/clickhouse.py b/posthog/temporal/common/clickhouse.py
@@ -417,6 +417,7 @@ async def get_client(
                 password=settings.CLICKHOUSE_PASSWORD,
                 database=settings.CLICKHOUSE_DATABASE,
                 max_execution_time=settings.CLICKHOUSE_MAX_EXECUTION_TIME,
+                max_memory_usage=settings.CLICKHOUSE_MAX_MEMORY_USAGE,
                 max_block_size=max_block_size,
                 output_format_arrow_string_as_string="true",
                 **kwargs,