From 4205aa9cb3ef3b314e52b22d825a3b01a1e78ac8 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 7 Aug 2024 10:38:49 +0330 Subject: [PATCH 1/2] fix: wrong kwarg given for FetchPlatforms! --- dags/discourse_analyzer_etl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/discourse_analyzer_etl.py b/dags/discourse_analyzer_etl.py index 714b4122..165073af 100644 --- a/dags/discourse_analyzer_etl.py +++ b/dags/discourse_analyzer_etl.py @@ -173,7 +173,7 @@ def analyze_discourse(platform_processed: dict[str, str | bool]) -> None: ``` """ logging.info(f"platform_processed: {platform_processed}") - fetcher = FetchPlatforms(plaform_name="discourse") + fetcher = FetchPlatforms(platform_name="discourse") platform_id = platform_processed["platform_id"] recompute = platform_processed["recompute"] From 76e76cc4dcd1e074981ae48ae43a8f89a7855ad7 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 7 Aug 2024 10:44:15 +0330 Subject: [PATCH 2/2] feat: Added more logs for better tracebility of work! --- dags/discourse_analyzer_etl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dags/discourse_analyzer_etl.py b/dags/discourse_analyzer_etl.py index 165073af..b590da8a 100644 --- a/dags/discourse_analyzer_etl.py +++ b/dags/discourse_analyzer_etl.py @@ -112,9 +112,12 @@ def discourse_etl_raw_data( transformed_data = transformer.transform( raw_data=extracted_data, ) - - loader = LoadTransformedData(platform_id=platform_id) - loader.load(processed_data=transformed_data, recompute=recompute) + if len(transformed_data) != 0: + logging.info(f"Loading {len(transformed_data)} transformed document in db!") + loader = LoadTransformedData(platform_id=platform_id) + loader.load(processed_data=transformed_data, recompute=recompute) + else: + logging.warning("No new document to load for discourse!") @task def discourse_etl_raw_members(