From 194bba7ecde248bfea651d389f0744cc5146028d Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 29 May 2024 20:33:39 +0330 Subject: [PATCH] feat: Added try catch for modules data fetching! --- .../src/utils/modules/gdrive.py | 35 ++++++++++------ .../src/utils/modules/github.py | 40 +++++++++++-------- .../src/utils/modules/mediawiki.py | 8 ++-- .../src/utils/modules/notion.py | 34 ++++++++++------ 4 files changed, 72 insertions(+), 45 deletions(-) diff --git a/dags/hivemind_etl_helpers/src/utils/modules/gdrive.py b/dags/hivemind_etl_helpers/src/utils/modules/gdrive.py index 4bc5a9bd..685637c5 100644 --- a/dags/hivemind_etl_helpers/src/utils/modules/gdrive.py +++ b/dags/hivemind_etl_helpers/src/utils/modules/gdrive.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from .modules_base import ModulesBase @@ -40,18 +41,26 @@ def get_learning_platforms(self): if platform["name"] != self.platform_name: continue - modules_options = platform["metadata"] - refresh_token = self.get_token( - platform_id=platform["platform"], token_type="google_refresh" - ) - platforms_data.append( - { - "community_id": str(community), - "drive_ids": modules_options.get("driveIds", []), - "folder_ids": modules_options.get("folderIds", []), - "file_ids": modules_options.get("fileIds", []), - "refresh_token": refresh_token, - } - ) + platform_id = platform["platform"] + + try: + modules_options = platform["metadata"] + refresh_token = self.get_token( + platform_id=platform_id, token_type="google_refresh" + ) + platforms_data.append( + { + "community_id": str(community), + "drive_ids": modules_options.get("driveIds", []), + "folder_ids": modules_options.get("folderIds", []), + "file_ids": modules_options.get("fileIds", []), + "refresh_token": refresh_token, + } + ) + except Exception as exp: + logging.error( + "Exception while fetching mediaWiki modules " + f"for platform: {platform_id} | exception: {exp}" + ) return platforms_data diff --git a/dags/hivemind_etl_helpers/src/utils/modules/github.py b/dags/hivemind_etl_helpers/src/utils/modules/github.py index b99ec0dc..dae7e45c 100644 --- a/dags/hivemind_etl_helpers/src/utils/modules/github.py +++ b/dags/hivemind_etl_helpers/src/utils/modules/github.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from .modules_base import ModulesBase @@ -41,22 +42,29 @@ def get_learning_platforms(self): continue platform_id = platform["platform"] - organization_id = self.get_platform_metadata( - platform_id=platform_id, - metadata_name="installationId", - ) - modules_options = platform["metadata"] - - # if github modules was activated - if modules_options["activated"] is True: - platforms_data.append( - { - "community_id": str(community), - "organization_ids": [organization_id], - # "repo_ids": modules_options.get("repoIds", []), - # "from_date": modules_options["fromDate"], - "from_date": None, - } + + try: + organization_id = self.get_platform_metadata( + platform_id=platform_id, + metadata_name="installationId", + ) + modules_options = platform["metadata"] + + # if github modules was activated + if modules_options["activated"] is True: + platforms_data.append( + { + "community_id": str(community), + "organization_ids": [organization_id], + # "repo_ids": modules_options.get("repoIds", []), + # "from_date": modules_options["fromDate"], + "from_date": None, + } + ) + except Exception as exp: + logging.error( + "Exception while fetching mediaWiki modules " + f"for platform: {platform_id} | exception: {exp}" ) return platforms_data diff --git a/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py b/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py index 86c677e5..c94ae611 100644 --- a/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py +++ b/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py @@ -39,14 +39,16 @@ def get_learning_platforms( if platform["name"] != self.platform_name: continue + platform_id = platform["platform"] + try: # TODO: retrieve baseURL and path in 1 db call base_url = self.get_platform_metadata( - platform_id=platform["platform"], + platform_id=platform_id, metadata_name="baseURL", ) path = self.get_platform_metadata( - platform_id=platform["platform"], + platform_id=platform_id, metadata_name="path", ) @@ -61,7 +63,7 @@ def get_learning_platforms( except Exception as exp: logging.error( "Exception while fetching mediaWiki modules " - f"for platform: {platform['platform']} | exception: {exp}" + f"for platform: {platform_id} | exception: {exp}" ) return communities_data diff --git a/dags/hivemind_etl_helpers/src/utils/modules/notion.py b/dags/hivemind_etl_helpers/src/utils/modules/notion.py index 67fbaa40..778a5ad5 100644 --- a/dags/hivemind_etl_helpers/src/utils/modules/notion.py +++ b/dags/hivemind_etl_helpers/src/utils/modules/notion.py @@ -1,3 +1,5 @@ +import logging + from .modules_base import ModulesBase @@ -38,18 +40,24 @@ def get_learning_platforms( if platform["name"] != self.platform_name: continue - modules_options = platform["metadata"] - token = self.get_token( - platform_id=platform["platform"], - token_type="notion_access", - ) - communities_data.append( - { - "community_id": str(community), - "database_ids": modules_options.get("databaseIds", []), - "page_ids": modules_options.get("pageIds", []), - "access_token": token, - } - ) + try: + modules_options = platform["metadata"] + token = self.get_token( + platform_id=platform["platform"], + token_type="notion_access", + ) + communities_data.append( + { + "community_id": str(community), + "database_ids": modules_options.get("databaseIds", []), + "page_ids": modules_options.get("pageIds", []), + "access_token": token, + } + ) + except Exception as exp: + logging.error( + "Exception while fetching mediaWiki modules " + f"for platform: {platform['platform']} | exception: {exp}" + ) return communities_data