Merge pull request #170 from TogetherCrew/fix/media-wiki-integration

feat: updated mediaWiki api url loading!
TogetherCrew · May 28, 2024 · 2943317 · 2943317
2 parents 7926881 + 28a1bf0
commit 2943317
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 4 deletions.
diff --git a/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py b/dags/hivemind_etl_helpers/src/utils/modules/mediawiki.py
@@ -37,16 +37,22 @@ def get_learning_platforms(
                 if platform["name"] != self.platform_name:
                     continue
 
+                # TODO: retrieve baseURL and path in 1 db call
                 base_url = self.get_platform_metadata(
                     platform_id=platform["platform"],
                     metadata_name="baseURL",
                 )
+                path = self.get_platform_metadata(
+                    platform_id=platform["platform"],
+                    metadata_name="path",
+                )
+
                 modules_options = platform["metadata"]
                 communities_data.append(
                     {
                         "community_id": str(community),
                         "page_titles": modules_options.get("pageIds", []),
-                        "base_url": base_url,
+                        "base_url": base_url + path,
                     }
                 )
 

diff --git a/dags/hivemind_etl_helpers/tests/integration/test_mediawiki_modules.py b/dags/hivemind_etl_helpers/tests/integration/test_mediawiki_modules.py
@@ -27,7 +27,8 @@ def test_get_single_data(self):
                 "_id": platform_id,
                 "name": "mediaWiki",
                 "metadata": {
-                    "baseURL": "http://example.com/api",
+                    "baseURL": "http://example.com",
+                    "path": "/api",
                 },
                 "community": community_id,
                 "disconnectedAt": None,
@@ -117,7 +118,8 @@ def test_get_mediawiki_communities_data_multiple_platforms(self):
                 "_id": platform_id1,
                 "name": "mediaWiki",
                 "metadata": {
-                    "baseURL": "http://example1.com/api",
+                    "baseURL": "http://example1.com",
+                    "path": "/api",
                 },
                 "community": community_id,
                 "disconnectedAt": None,
@@ -132,7 +134,8 @@ def test_get_mediawiki_communities_data_multiple_platforms(self):
                 "_id": platform_id2,
                 "name": "mediaWiki",
                 "metadata": {
-                    "baseURL": "http://example2.com/api",
+                    "baseURL": "http://example2.com",
+                    "path": "/api",
                 },
                 "community": community_id,
                 "disconnectedAt": None,