fix(data-warehouse): Clear the delta lake table cache after a pipelin…

…e has run (#27215)
PostHog · Jan 2, 2025 · b32b014 · b32b014
1 parent ea86a3c
commit b32b014
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/posthog/temporal/data_imports/pipelines/pipeline/pipeline.py b/posthog/temporal/data_imports/pipelines/pipeline/pipeline.py
@@ -100,8 +100,14 @@ def run(self):
             self._post_run_operations(row_count=row_count)
         finally:
             # Help reduce the memory footprint of each job
+            delta_table = self._delta_table_helper.get_delta_table()
+            self._delta_table_helper.get_delta_table.cache_clear()
+            if delta_table:
+                del delta_table
+
             del self._resource
             del self._delta_table_helper
+
             if "buffer" in locals() and buffer is not None:
                 del buffer
             if "py_table" in locals() and py_table is not None: