From 76e76cc4dcd1e074981ae48ae43a8f89a7855ad7 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 7 Aug 2024 10:44:15 +0330 Subject: [PATCH] feat: Added more logs for better tracebility of work! --- dags/discourse_analyzer_etl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dags/discourse_analyzer_etl.py b/dags/discourse_analyzer_etl.py index 165073af..b590da8a 100644 --- a/dags/discourse_analyzer_etl.py +++ b/dags/discourse_analyzer_etl.py @@ -112,9 +112,12 @@ def discourse_etl_raw_data( transformed_data = transformer.transform( raw_data=extracted_data, ) - - loader = LoadTransformedData(platform_id=platform_id) - loader.load(processed_data=transformed_data, recompute=recompute) + if len(transformed_data) != 0: + logging.info(f"Loading {len(transformed_data)} transformed document in db!") + loader = LoadTransformedData(platform_id=platform_id) + loader.load(processed_data=transformed_data, recompute=recompute) + else: + logging.warning("No new document to load for discourse!") @task def discourse_etl_raw_members(