From df0b3f31230639f82635a979b9f4bcdd0471f1e1 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Sun, 8 Dec 2024 12:32:11 +0330 Subject: [PATCH 1/3] feat: Added logs to identify which platform is being processed! --- dags/discourse_analyzer_etl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dags/discourse_analyzer_etl.py b/dags/discourse_analyzer_etl.py index e296e961..738c0da2 100644 --- a/dags/discourse_analyzer_etl.py +++ b/dags/discourse_analyzer_etl.py @@ -106,6 +106,8 @@ def discourse_etl_raw_data( } ``` """ + logging.info(f"Processing platform: {platform_info}") + platform_id = platform_info["platform_id"] forum_endpoint = platform_info["id"] period = platform_info["period"] @@ -147,6 +149,7 @@ def discourse_etl_raw_members( } ``` """ + logging.info(f"Processing platform: {platform_info}") platform_id = platform_info["platform_id"] forum_endpoint = platform_info["id"] # period = platform_info["period"] From 07f34c0137135a740aeba7cdb3f877689e549669 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Sun, 8 Dec 2024 13:31:52 +0330 Subject: [PATCH 2/3] feat: Added raw data preparation progress log! --- dags/analyzer_helper/discourse/transform_raw_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dags/analyzer_helper/discourse/transform_raw_data.py b/dags/analyzer_helper/discourse/transform_raw_data.py index 85fd15b5..a561529a 100644 --- a/dags/analyzer_helper/discourse/transform_raw_data.py +++ b/dags/analyzer_helper/discourse/transform_raw_data.py @@ -1,3 +1,5 @@ +import logging + from analyzer_helper.discourse.utils.convert_date_time_formats import ( DateTimeFormatConverter, ) @@ -90,7 +92,7 @@ def create_data_entry( def transform(self, raw_data: list) -> list: transformed_data = [] - for entry in raw_data: + for idx, entry in enumerate(raw_data): # Create main post entry transformed_data.append(self.create_data_entry(entry)) @@ -115,4 +117,6 @@ def transform(self, raw_data: list) -> list: ) # TODO: Create entry for mentioned users + logging.info(f"Preparing raw data: {idx + 1}/{len(raw_data)}") + return transformed_data From 20a022293fc251e32612e9b0eb4d5ce4a4014d6f Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Sun, 8 Dec 2024 13:32:48 +0330 Subject: [PATCH 3/3] feat: Added raw members progress logs! --- dags/analyzer_helper/discourse/transform_raw_members.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dags/analyzer_helper/discourse/transform_raw_members.py b/dags/analyzer_helper/discourse/transform_raw_members.py index 9a5d6398..f1bff0b5 100644 --- a/dags/analyzer_helper/discourse/transform_raw_members.py +++ b/dags/analyzer_helper/discourse/transform_raw_members.py @@ -19,10 +19,12 @@ def transform(self, raw_members: list) -> list: """ transformed_members = [] - for member in raw_members: + for idx, member in enumerate(raw_members): try: transformed_member = self.transform_member(member=member) transformed_members.append(transformed_member) + + logging.info(f"Preparing raw members: {idx + 1}/{len(raw_members)}") except Exception as e: logging.error(f"Error transforming raw discourse member {member}: {e}")