feat: Added more logs for better tracebility of work!

TogetherCrew · Aug 7, 2024 · 76e76cc · 76e76cc
1 parent 4205aa9
commit 76e76cc
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/dags/discourse_analyzer_etl.py b/dags/discourse_analyzer_etl.py
@@ -112,9 +112,12 @@ def discourse_etl_raw_data(
         transformed_data = transformer.transform(
             raw_data=extracted_data,
         )
-
-        loader = LoadTransformedData(platform_id=platform_id)
-        loader.load(processed_data=transformed_data, recompute=recompute)
+        if len(transformed_data) != 0:
+            logging.info(f"Loading {len(transformed_data)} transformed document in db!")
+            loader = LoadTransformedData(platform_id=platform_id)
+            loader.load(processed_data=transformed_data, recompute=recompute)
+        else:
+            logging.warning("No new document to load for discourse!")
 
     @task
     def discourse_etl_raw_members(