Merge pull request #328 from TogetherCrew/feat/327-use-hivemind-backe…

…nd-lib Feat/327 use hivemind backend lib
TogetherCrew · Nov 21, 2024 · c6db4c1 · c6db4c1
2 parents 39a5a73 + b51d5ca
commit c6db4c1
Show file tree

Hide file tree

Showing 81 changed files with 69 additions and 1,002 deletions.
diff --git a/dags/analyzer_helper/common/fetch_platforms.py b/dags/analyzer_helper/common/fetch_platforms.py
@@ -1,5 +1,5 @@
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class FetchPlatforms:

diff --git a/dags/analyzer_helper/common/load_transformed_data.py b/dags/analyzer_helper/common/load_transformed_data.py
@@ -3,7 +3,7 @@
 from analyzer_helper.common.base.load_transformed_data_base import (
     LoadTransformedDataBase,
 )
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class LoadTransformedData(LoadTransformedDataBase):

diff --git a/dags/analyzer_helper/common/load_transformed_members.py b/dags/analyzer_helper/common/load_transformed_members.py
@@ -3,7 +3,7 @@
 from analyzer_helper.common.base.load_transformed_members_base import (
     LoadTransformedMembersBase,
 )
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class LoadTransformedMembers(LoadTransformedMembersBase):

diff --git a/dags/analyzer_helper/discord/discord_extract_raw_infos.py b/dags/analyzer_helper/discord/discord_extract_raw_infos.py
@@ -2,7 +2,7 @@
 from datetime import datetime, timezone
 
 from analyzer_helper.discord.extract_raw_info_base import ExtractRawInfosBase
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class DiscordExtractRawInfos(ExtractRawInfosBase):

diff --git a/dags/analyzer_helper/discord/discord_extract_raw_members.py b/dags/analyzer_helper/discord/discord_extract_raw_members.py
@@ -1,7 +1,7 @@
 import logging
 
 from analyzer_helper.discord.extract_raw_member_base import ExtractRawMembersBase
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class DiscordExtractRawMembers(ExtractRawMembersBase):

diff --git a/dags/analyzer_helper/discord/discord_load_transformed_data.py b/dags/analyzer_helper/discord/discord_load_transformed_data.py
@@ -1,7 +1,7 @@
 import logging
 
 from analyzer_helper.discord.load_transformed_data_base import LoadTransformedDataBase
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class DiscordLoadTransformedData(LoadTransformedDataBase):

diff --git a/dags/analyzer_helper/discord/discord_load_transformed_members.py b/dags/analyzer_helper/discord/discord_load_transformed_members.py
@@ -3,7 +3,7 @@
 from analyzer_helper.discord.load_transformed_members_base import (
     LoadTransformedMembersBase,
 )
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class DiscordLoadTransformedMembers(LoadTransformedMembersBase):

diff --git a/dags/analyzer_helper/discord/fetch_discord_platforms.py b/dags/analyzer_helper/discord/fetch_discord_platforms.py
@@ -1,5 +1,5 @@
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class FetchDiscordPlatforms:

diff --git a/dags/analyzer_helper/discord/utils/is_user_bot.py b/dags/analyzer_helper/discord/utils/is_user_bot.py
@@ -1,4 +1,4 @@
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class UserBotChecker:

diff --git a/dags/analyzer_helper/discourse/extract_raw_data.py b/dags/analyzer_helper/discourse/extract_raw_data.py
@@ -5,7 +5,7 @@
     DateTimeFormatConverter,
 )
 from github.neo4j_storage.neo4j_connection import Neo4jConnection
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class ExtractRawInfo:

diff --git a/dags/analyzer_helper/discourse/extract_raw_members.py b/dags/analyzer_helper/discourse/extract_raw_members.py
@@ -4,7 +4,7 @@
     DateTimeFormatConverter,
 )
 from github.neo4j_storage.neo4j_connection import Neo4jConnection
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class ExtractRawMembers:

diff --git a/dags/analyzer_helper/telegram/extract_raw_data.py b/dags/analyzer_helper/telegram/extract_raw_data.py
@@ -5,7 +5,7 @@
     DateTimeFormatConverter,
 )
 from github.neo4j_storage.neo4j_connection import Neo4jConnection
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class ExtractRawInfo:

diff --git a/dags/analyzer_helper/telegram/extract_raw_members.py b/dags/analyzer_helper/telegram/extract_raw_members.py
@@ -2,8 +2,8 @@
     DateTimeFormatConverter,
 )
 from github.neo4j_storage.neo4j_connection import Neo4jConnection
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
 from pymongo import DESCENDING
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class ExtractRawMembers:

diff --git a/dags/analyzer_helper/telegram/tests/integration/test_telegram_extract_raw_data.py b/dags/analyzer_helper/telegram/tests/integration/test_telegram_extract_raw_data.py
@@ -3,7 +3,7 @@
 
 from analyzer_helper.telegram.extract_raw_data import ExtractRawInfo
 from github.neo4j_storage.neo4j_connection import Neo4jConnection
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestExtractRawInfo(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discord_extract_raw_info.py b/dags/analyzer_helper/tests/integration/test_discord_extract_raw_info.py
@@ -3,7 +3,7 @@
 
 from analyzer_helper.discord.discord_extract_raw_infos import DiscordExtractRawInfos
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestDiscordExtractRawInfos(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discord_extract_raw_members.py b/dags/analyzer_helper/tests/integration/test_discord_extract_raw_members.py
@@ -2,7 +2,7 @@
 from datetime import datetime
 
 from analyzer_helper.discord.discord_extract_raw_members import DiscordExtractRawMembers
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestDiscordExtractRawMembers(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discord_is_user_bot.py b/dags/analyzer_helper/tests/integration/test_discord_is_user_bot.py
@@ -3,7 +3,7 @@
 
 from analyzer_helper.discord.utils.is_user_bot import UserBotChecker
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestUserBotChecker(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discord_load_transformed_data.py b/dags/analyzer_helper/tests/integration/test_discord_load_transformed_data.py
@@ -4,7 +4,7 @@
 from analyzer_helper.discord.discord_load_transformed_data import (
     DiscordLoadTransformedData,
 )
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestDiscordLoadTransformedData(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discord_load_transformed_members.py b/dags/analyzer_helper/tests/integration/test_discord_load_transformed_members.py
@@ -4,7 +4,7 @@
 from analyzer_helper.discord.discord_load_transformed_members import (
     DiscordLoadTransformedMembers,
 )
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestDiscordLoadTransformedMembers(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discord_transform_raw_data.py b/dags/analyzer_helper/tests/integration/test_discord_transform_raw_data.py
@@ -4,7 +4,7 @@
 from analyzer_helper.discord.discord_transform_raw_data import DiscordTransformRawData
 from analyzer_helper.discord.utils.is_user_bot import UserBotChecker
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestDiscordTransformRawData(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_discourse_extract_raw_data.py b/dags/analyzer_helper/tests/integration/test_discourse_extract_raw_data.py
@@ -3,7 +3,7 @@
 
 from analyzer_helper.discourse.extract_raw_data import ExtractRawInfo
 from github.neo4j_storage.neo4j_connection import Neo4jConnection
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestExtractRawInfo(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/integration/test_integration_fetch_discord_platforms.py b/dags/analyzer_helper/tests/integration/test_integration_fetch_discord_platforms.py
@@ -3,7 +3,7 @@
 
 from analyzer_helper.discord.fetch_discord_platforms import FetchDiscordPlatforms
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestFetchDiscordPlatforms(unittest.TestCase):

diff --git a/dags/analyzer_helper/tests/unit/test_unit_fetch_discord_platforms.py b/dags/analyzer_helper/tests/unit/test_unit_fetch_discord_platforms.py
@@ -4,7 +4,7 @@
 
 from analyzer_helper.discord.fetch_discord_platforms import FetchDiscordPlatforms
 from bson import ObjectId
-from hivemind_etl_helpers.src.utils.mongo import MongoSingleton
+from tc_hivemind_backend.db.mongo import MongoSingleton
 
 
 class TestFetchDiscordPlatformsUnit(unittest.TestCase):

diff --git a/dags/hivemind_etl_helpers/github_etl.py b/dags/hivemind_etl_helpers/github_etl.py
@@ -2,7 +2,6 @@
 from datetime import datetime
 
 from dotenv import load_dotenv
-from hivemind_etl_helpers.ingestion_pipeline import CustomIngestionPipeline
 from hivemind_etl_helpers.src.db.github.extract import (
     GithubExtraction,
     fetch_issues,
@@ -13,6 +12,7 @@
 )
 from hivemind_etl_helpers.src.db.github.transform import GitHubTransformation
 from llama_index.core import Document
+from tc_hivemind_backend.ingest_qdrant import CustomIngestionPipeline
 
 
 def process_github_vectorstore(