feat: Using the shared library!

We have to update the test cases too.

dags/hivemind_etl_helpers/discord_mongo_summary_etl.py

@@ -6,9 +6,9 @@
     find_guild_id_by_community_id,
 )
 from hivemind_etl_helpers.src.document_node_parser import configure_node_parser
-from hivemind_etl_helpers.src.utils.cohere_embedding import CohereEmbedding
-from hivemind_etl_helpers.src.utils.load_llm_params import load_model_hyperparams
-from hivemind_etl_helpers.src.utils.pg_db_utils import setup_db
+from tc_hivemind_backend.embeddings.cohere import CohereEmbedding
+from tc_hivemind_backend.db.utils.model_hyperparams import load_model_hyperparams
+from tc_hivemind_backend.db.pg_db_utils import setup_db
 from hivemind_etl_helpers.src.utils.pg_vector_access import PGVectorAccess
 from llama_index.response_synthesizers import get_response_synthesizer
 

dags/hivemind_etl_helpers/discord_mongo_vector_store_etl.py

@@ -9,9 +9,9 @@
     find_guild_id_by_community_id,
 )
 from hivemind_etl_helpers.src.document_node_parser import configure_node_parser
-from hivemind_etl_helpers.src.utils.cohere_embedding import CohereEmbedding
-from hivemind_etl_helpers.src.utils.load_llm_params import load_model_hyperparams
-from hivemind_etl_helpers.src.utils.pg_db_utils import setup_db
+from tc_hivemind_backend.embeddings.cohere import CohereEmbedding
+from tc_hivemind_backend.db.utils.model_hyperparams import load_model_hyperparams
+from tc_hivemind_backend.db.pg_db_utils import setup_db
 from hivemind_etl_helpers.src.utils.pg_vector_access import PGVectorAccess
 
 

dags/hivemind_etl_helpers/discourse_summary_etl.py

@@ -9,9 +9,9 @@
 )
 from hivemind_etl_helpers.src.db.discourse.utils.get_forums import get_forums
 from hivemind_etl_helpers.src.document_node_parser import configure_node_parser
-from hivemind_etl_helpers.src.utils.cohere_embedding import CohereEmbedding
-from hivemind_etl_helpers.src.utils.load_llm_params import load_model_hyperparams
-from hivemind_etl_helpers.src.utils.pg_db_utils import setup_db
+from tc_hivemind_backend.embeddings.cohere import CohereEmbedding
+from tc_hivemind_backend.db.utils.model_hyperparams import load_model_hyperparams
+from tc_hivemind_backend.db.pg_db_utils import setup_db
 from hivemind_etl_helpers.src.utils.pg_vector_access import PGVectorAccess
 from llama_index import Document
 from llama_index.response_synthesizers import get_response_synthesizer

dags/hivemind_etl_helpers/discourse_vectorstore_etl.py

@@ -7,9 +7,9 @@
 from hivemind_etl_helpers.src.db.discourse.utils.get_forums import get_forums
 from hivemind_etl_helpers.src.document_node_parser import configure_node_parser
 from hivemind_etl_helpers.src.utils.check_documents import check_documents
-from hivemind_etl_helpers.src.utils.cohere_embedding import CohereEmbedding
-from hivemind_etl_helpers.src.utils.load_llm_params import load_model_hyperparams
-from hivemind_etl_helpers.src.utils.pg_db_utils import setup_db
+from tc_hivemind_backend.embeddings.cohere import CohereEmbedding
+from tc_hivemind_backend.db.utils.model_hyperparams import load_model_hyperparams
+from tc_hivemind_backend.db.pg_db_utils import setup_db
 from hivemind_etl_helpers.src.utils.pg_vector_access import PGVectorAccess
 
 

dags/hivemind_etl_helpers/gdrive_etl.py

@@ -8,8 +8,8 @@
 )
 from hivemind_etl_helpers.src.document_node_parser import configure_node_parser
 from hivemind_etl_helpers.src.utils.check_documents import check_documents
-from hivemind_etl_helpers.src.utils.cohere_embedding import CohereEmbedding
-from hivemind_etl_helpers.src.utils.load_llm_params import load_model_hyperparams
+from tc_hivemind_backend.embeddings.cohere import CohereEmbedding
+from tc_hivemind_backend.db.utils.model_hyperparams import load_model_hyperparams
 from hivemind_etl_helpers.src.utils.pg_vector_access import PGVectorAccess
 
 

dags/hivemind_etl_helpers/src/db/gdrive/db_utils.py

@@ -3,8 +3,8 @@
 
 import psycopg2
 from dateutil import parser
-from hivemind_etl_helpers.src.utils.pg_db_utils import convert_tuple_str
-from hivemind_etl_helpers.src.utils.postgres import PostgresSingleton
+from tc_hivemind_backend.db.pg_db_utils import convert_tuple_str
+from tc_hivemind_backend.db.postgresql import PostgresSingleton
 
 
 def setup_db(community_id: str) -> None:

dags/hivemind_etl_helpers/src/db/gdrive/delete_records.py

@@ -1,7 +1,7 @@
 import logging
 
 from hivemind_etl_helpers.src.utils.pg_db_utils import convert_tuple_str
-from hivemind_etl_helpers.src.utils.postgres import PostgresSingleton
+from tc_hivemind_backend.db.postgresql import PostgresSingleton
 
 
 def delete_records(db_name: str, table_name: str, metadata_file_id: list[str]) -> None:

dags/hivemind_etl_helpers/src/utils/credentials.py

@@ -27,27 +27,3 @@ def load_mongo_credentials() -> dict[str, str]:
     mongo_creds["port"] = os.getenv("MONGODB_PORT", "")
 
     return mongo_creds
-
-
-def load_postgres_credentials() -> dict[str, str]:
-    """
-    load postgres credentials into a dictionary
-
-    Returns
-    ---------
-    credentials : dict[str, str]
-        a dictionary containing
-        `host`, `password`, `user`, `port`, and `db_name` as keys
-        and values are the values for the credentials
-    """
-    load_dotenv()
-
-    credentials: dict[str, str] = {}
-
-    credentials["host"] = os.getenv("POSTGRES_HOST", "")
-    credentials["password"] = os.getenv("POSTGRES_PASS", "")
-    credentials["user"] = os.getenv("POSTGRES_USER", "")
-    credentials["port"] = os.getenv("POSTGRES_PORT", "")
-    credentials["db_name"] = os.getenv("POSTGRES_DBNAME", "")
-
-    return credentials

dags/hivemind_etl_helpers/src/utils/pg_vector_access.py

@@ -1,9 +1,9 @@
 import logging
 import time
 
-from hivemind_etl_helpers.src.utils.credentials import load_postgres_credentials
-from hivemind_etl_helpers.src.utils.load_llm_params import load_model_hyperparams
-from hivemind_etl_helpers.src.utils.pg_db_utils import delete_data
+from tc_hivemind_backend.db.credentials import load_postgres_credentials
+from tc_hivemind_backend.db.utils.model_hyperparams import load_model_hyperparams
+from tc_hivemind_backend.db.utils.delete_data import delete_data
 from llama_index import Document, MockEmbedding, ServiceContext, StorageContext
 from llama_index.embeddings import BaseEmbedding, OpenAIEmbedding
 from llama_index.indices.vector_store import VectorStoreIndex