chore(data-warehouse): Adjusted temporal and sql settings for long ru…

…nning tasks (#25196)
PostHog · Sep 25, 2024 · de87079 · de87079
1 parent 81d0863
commit de87079
Show file tree

Hide file tree

Showing 3 changed files with 320 additions and 322 deletions.
diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py
@@ -201,13 +201,13 @@ async def run(self, inputs: ExternalDataWorkflowInputs):
             timeout_params = (
                 {"start_to_close_timeout": dt.timedelta(weeks=1), "retry_policy": RetryPolicy(maximum_attempts=1)}
                 if incremental
-                else {"start_to_close_timeout": dt.timedelta(hours=5), "retry_policy": RetryPolicy(maximum_attempts=3)}
+                else {"start_to_close_timeout": dt.timedelta(hours=12), "retry_policy": RetryPolicy(maximum_attempts=3)}
             )
 
             await workflow.execute_activity(
                 import_data_activity,
                 job_inputs,
-                heartbeat_timeout=dt.timedelta(minutes=2),
+                heartbeat_timeout=dt.timedelta(minutes=5),
                 **timeout_params,
             )  # type: ignore
 

diff --git a/posthog/temporal/data_imports/pipelines/sql_database/settings.py b/posthog/temporal/data_imports/pipelines/sql_database/settings.py
@@ -1,3 +1,3 @@
 """Sql Database source settings and constants"""
 
-DEFAULT_CHUNK_SIZE = 1000
+DEFAULT_CHUNK_SIZE = 5000