From 9b9118943ee7d5e9697d8cc0e0dcf24b8f3f89ea Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Thu, 14 Nov 2024 12:46:30 +0000 Subject: [PATCH] fix(data-warehouse): Use the correct DLT setting (#26187) --- posthog/temporal/data_imports/pipelines/pipeline_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/temporal/data_imports/pipelines/pipeline_sync.py b/posthog/temporal/data_imports/pipelines/pipeline_sync.py index 05ccedf2fddff..19c35d6cdf5b0 100644 --- a/posthog/temporal/data_imports/pipelines/pipeline_sync.py +++ b/posthog/temporal/data_imports/pipelines/pipeline_sync.py @@ -163,7 +163,7 @@ def _iter_chunks(self, lst: list[Any], n: int) -> Iterator[list[Any]]: dlt.config["data_writer.file_max_items"] = 500_000 dlt.config["data_writer.file_max_bytes"] = 500_000_000 # 500 MB - dlt.config["loader_parallelism_strategy"] = "table-sequential" + dlt.config["parallelism_strategy"] = "table-sequential" dlt.config["delta_jobs_per_write"] = 1 dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True