feat: Added logs to identify which platform is being processed!

dags/analyzer_helper/discourse/transform_raw_data.py

@@ -1,3 +1,5 @@
+import logging
+
 from analyzer_helper.discourse.utils.convert_date_time_formats import (
     DateTimeFormatConverter,
 )
@@ -90,7 +92,7 @@ def create_data_entry(
 
     def transform(self, raw_data: list) -> list:
         transformed_data = []
-        for entry in raw_data:
+        for idx, entry in enumerate(raw_data):
             # Create main post entry
             transformed_data.append(self.create_data_entry(entry))
 
@@ -115,4 +117,6 @@ def transform(self, raw_data: list) -> list:
                 )
             # TODO: Create entry for mentioned users
 
+            logging.info(f"Preparing raw data: {idx + 1}/{len(raw_data)}")
+
         return transformed_data

dags/analyzer_helper/discourse/transform_raw_members.py

@@ -19,10 +19,12 @@ def transform(self, raw_members: list) -> list:
         """
         transformed_members = []
 
-        for member in raw_members:
+        for idx, member in enumerate(raw_members):
             try:
                 transformed_member = self.transform_member(member=member)
                 transformed_members.append(transformed_member)
+
+                logging.info(f"Preparing raw members: {idx + 1}/{len(raw_members)}")
             except Exception as e:
                 logging.error(f"Error transforming raw discourse member {member}: {e}")
 

dags/discourse_analyzer_etl.py

@@ -106,6 +106,8 @@ def discourse_etl_raw_data(
             }
             ```
         """
+        logging.info(f"Processing platform: {platform_info}")
+
         platform_id = platform_info["platform_id"]
         forum_endpoint = platform_info["id"]
         period = platform_info["period"]
@@ -147,6 +149,7 @@ def discourse_etl_raw_members(
             }
             ```
         """
+        logging.info(f"Processing platform: {platform_info}")
         platform_id = platform_info["platform_id"]
         forum_endpoint = platform_info["id"]
         # period = platform_info["period"]