From 0c1f1a72db45109a3c76a0f3c457f88d62674e33 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 28 May 2024 15:16:31 +0330 Subject: [PATCH 1/2] feat: updated mediaWiki modules data fetching! Updated to support both baseURL and path. --- dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py b/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py index cd799fba..0a44e938 100644 --- a/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py +++ b/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py @@ -37,16 +37,22 @@ def get_learning_platforms( if platform["name"] != self.platform_name: continue + # TODO: retrieve baseURL and path in 1 db call base_url = self.get_platform_metadata( platform_id=platform["platform"], metadata_name="baseURL", ) + path = self.get_platform_metadata( + platform_id=platform["platform"], + metadata_name="path", + ) + modules_options = platform["metadata"] communities_data.append( { "community_id": str(community), "page_titles": modules_options.get("pageIds", []), - "base_url": base_url, + "base_url": base_url + path, } ) From 28a1bf04120c494e797c965ca89379199989fdb8 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 28 May 2024 15:20:12 +0330 Subject: [PATCH 2/2] fix: add support for mediaWiki api path in test cases! --- .../tests/integration/test_mediawiki_modules.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dags/hivemind_etl_helpers/tests/integration/test_mediawiki_modules.py b/dags/hivemind_etl_helpers/tests/integration/test_mediawiki_modules.py index 83783242..8e6152a4 100644 --- a/dags/hivemind_etl_helpers/tests/integration/test_mediawiki_modules.py +++ b/dags/hivemind_etl_helpers/tests/integration/test_mediawiki_modules.py @@ -27,7 +27,8 @@ def test_get_single_data(self): "_id": platform_id, "name": "mediaWiki", "metadata": { - "baseURL": "http://example.com/api", + "baseURL": "http://example.com", + "path": "/api", }, "community": community_id, "disconnectedAt": None, @@ -117,7 +118,8 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "_id": platform_id1, "name": "mediaWiki", "metadata": { - "baseURL": "http://example1.com/api", + "baseURL": "http://example1.com", + "path": "/api", }, "community": community_id, "disconnectedAt": None, @@ -132,7 +134,8 @@ def test_get_mediawiki_communities_data_multiple_platforms(self): "_id": platform_id2, "name": "mediaWiki", "metadata": { - "baseURL": "http://example2.com/api", + "baseURL": "http://example2.com", + "path": "/api", }, "community": community_id, "disconnectedAt": None,