chore(data-warehouse): Delete folder when source is deleted (#19061)

use right data job
PostHog · Dec 4, 2023 · a1237d2 · a1237d2
1 parent 8b09acd
commit a1237d2
Showing 1 changed file with 14 additions and 5 deletions.
diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py
@@ -122,17 +122,26 @@ def prefix_exists(self, source_type: str, prefix: str) -> bool:
     def destroy(self, request: Request, *args: Any, **kwargs: Any) -> Response:
         instance = self.get_object()
 
-        latest_job = (
+        latest_running_job = (
             ExternalDataJob.objects.filter(pipeline_id=instance.pk, team_id=instance.team_id)
             .order_by("-created_at")
             .first()
         )
-        if latest_job and latest_job.workflow_id and latest_job.status == "Running":
-            cancel_external_data_workflow(latest_job.workflow_id)
+        if latest_running_job and latest_running_job.workflow_id and latest_running_job.status == "Running":
+            cancel_external_data_workflow(latest_running_job.workflow_id)
+
+        latest_completed_job = (
+            ExternalDataJob.objects.filter(pipeline_id=instance.pk, team_id=instance.team_id, status="Completed")
+            .order_by("-created_at")
+            .first()
+        )
+        if latest_completed_job:
             try:
-                delete_data_import_folder(latest_job.folder_path)
+                delete_data_import_folder(latest_completed_job.folder_path)
             except Exception as e:
-                logger.exception(f"Could not clean up data import folder: {latest_job.folder_path}", exc_info=e)
+                logger.exception(
+                    f"Could not clean up data import folder: {latest_completed_job.folder_path}", exc_info=e
+                )
                 pass
 
         delete_external_data_schedule(instance)