feat: Added try catch for modules data fetching!

TogetherCrew · May 29, 2024 · 194bba7 · 194bba7
1 parent 8c45afe
commit 194bba7
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 45 deletions.
diff --git a/dags/hivemind_etl_helpers/src/utils/modules/gdrive.py b/dags/hivemind_etl_helpers/src/utils/modules/gdrive.py
@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 from .modules_base import ModulesBase
@@ -40,18 +41,26 @@ def get_learning_platforms(self):
                 if platform["name"] != self.platform_name:
                     continue
 
-                modules_options = platform["metadata"]
-                refresh_token = self.get_token(
-                    platform_id=platform["platform"], token_type="google_refresh"
-                )
-                platforms_data.append(
-                    {
-                        "community_id": str(community),
-                        "drive_ids": modules_options.get("driveIds", []),
-                        "folder_ids": modules_options.get("folderIds", []),
-                        "file_ids": modules_options.get("fileIds", []),
-                        "refresh_token": refresh_token,
-                    }
-                )
+                platform_id = platform["platform"]
+
+                try:
+                    modules_options = platform["metadata"]
+                    refresh_token = self.get_token(
+                        platform_id=platform_id, token_type="google_refresh"
+                    )
+                    platforms_data.append(
+                        {
+                            "community_id": str(community),
+                            "drive_ids": modules_options.get("driveIds", []),
+                            "folder_ids": modules_options.get("folderIds", []),
+                            "file_ids": modules_options.get("fileIds", []),
+                            "refresh_token": refresh_token,
+                        }
+                    )
+                except Exception as exp:
+                    logging.error(
+                        "Exception while fetching mediaWiki modules "
+                        f"for platform: {platform_id} | exception: {exp}"
+                    )
 
         return platforms_data
diff --git a/dags/hivemind_etl_helpers/src/utils/modules/github.py b/dags/hivemind_etl_helpers/src/utils/modules/github.py
@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 from .modules_base import ModulesBase
@@ -41,22 +42,29 @@ def get_learning_platforms(self):
                     continue
 
                 platform_id = platform["platform"]
-                organization_id = self.get_platform_metadata(
-                    platform_id=platform_id,
-                    metadata_name="installationId",
-                )
-                modules_options = platform["metadata"]
-
-                # if github modules was activated
-                if modules_options["activated"] is True:
-                    platforms_data.append(
-                        {
-                            "community_id": str(community),
-                            "organization_ids": [organization_id],
-                            # "repo_ids": modules_options.get("repoIds", []),
-                            # "from_date": modules_options["fromDate"],
-                            "from_date": None,
-                        }
+
+                try:
+                    organization_id = self.get_platform_metadata(
+                        platform_id=platform_id,
+                        metadata_name="installationId",
+                    )
+                    modules_options = platform["metadata"]
+
+                    # if github modules was activated
+                    if modules_options["activated"] is True:
+                        platforms_data.append(
+                            {
+                                "community_id": str(community),
+                                "organization_ids": [organization_id],
+                                # "repo_ids": modules_options.get("repoIds", []),
+                                # "from_date": modules_options["fromDate"],
+                                "from_date": None,
+                            }
+                        )
+                except Exception as exp:
+                    logging.error(
+                        "Exception while fetching mediaWiki modules "
+                        f"for platform: {platform_id} | exception: {exp}"
                     )
 
         return platforms_data
diff --git a/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py b/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py
@@ -39,14 +39,16 @@ def get_learning_platforms(
                 if platform["name"] != self.platform_name:
                     continue
 
+                platform_id = platform["platform"]
+
                 try:
                     # TODO: retrieve baseURL and path in 1 db call
                     base_url = self.get_platform_metadata(
-                        platform_id=platform["platform"],
+                        platform_id=platform_id,
                         metadata_name="baseURL",
                     )
                     path = self.get_platform_metadata(
-                        platform_id=platform["platform"],
+                        platform_id=platform_id,
                         metadata_name="path",
                     )
 
@@ -61,7 +63,7 @@ def get_learning_platforms(
                 except Exception as exp:
                     logging.error(
                         "Exception while fetching mediaWiki modules "
-                        f"for platform: {platform['platform']} | exception: {exp}"
+                        f"for platform: {platform_id} | exception: {exp}"
                     )
 
         return communities_data
diff --git a/dags/hivemind_etl_helpers/src/utils/modules/notion.py b/dags/hivemind_etl_helpers/src/utils/modules/notion.py
@@ -1,3 +1,5 @@
+import logging
+
 from .modules_base import ModulesBase
 
 
@@ -38,18 +40,24 @@ def get_learning_platforms(
                 if platform["name"] != self.platform_name:
                     continue
 
-                modules_options = platform["metadata"]
-                token = self.get_token(
-                    platform_id=platform["platform"],
-                    token_type="notion_access",
-                )
-                communities_data.append(
-                    {
-                        "community_id": str(community),
-                        "database_ids": modules_options.get("databaseIds", []),
-                        "page_ids": modules_options.get("pageIds", []),
-                        "access_token": token,
-                    }
-                )
+                try:
+                    modules_options = platform["metadata"]
+                    token = self.get_token(
+                        platform_id=platform["platform"],
+                        token_type="notion_access",
+                    )
+                    communities_data.append(
+                        {
+                            "community_id": str(community),
+                            "database_ids": modules_options.get("databaseIds", []),
+                            "page_ids": modules_options.get("pageIds", []),
+                            "access_token": token,
+                        }
+                    )
+                except Exception as exp:
+                    logging.error(
+                        "Exception while fetching mediaWiki modules "
+                        f"for platform: {platform['platform']} | exception: {exp}"
+                    )
 
         return communities_data