Merge pull request #158 from TogetherCrew/feat/add-logs

feat: Add more logs!

dags/hivemind_etl_helpers/debugs.py

@@ -0,0 +1,8 @@
+import logging
+
+from tc_hivemind_backend.db.credentials import load_qdrant_credentials
+
+
+def pring_qdrant_creds():
+    qdrant_creds = load_qdrant_credentials()
+    logging.info(f"qdrant_creds: {qdrant_creds}")

dags/hivemind_etl_helpers/ingestion_pipeline.py

@@ -39,6 +39,9 @@ def __init__(self, community_id: str, collection_name: str, testing: bool = Fals
 
     def run_pipeline(self, docs: list[Document]):
         # qdrant is just collection based and doesn't have any database
+        logging.info(
+            f"{len(docs)} docuemnts was extracted and now loading into QDrant DB!"
+        )
         qdrant_collection_name = f"{self.collection_name}_{self.platform_name}"
         vector_access = QDrantVectorAccess(collection_name=qdrant_collection_name)
         vector_store = vector_access.setup_qdrant_vector_store()
@@ -61,6 +64,7 @@ def run_pipeline(self, docs: list[Document]):
             ),
             docstore_strategy=DocstoreStrategy.UPSERTS,
         )
+        logging.info("Pipeline created, now inserting documents into pipeline!")
         try:
             nodes = pipeline.run(documents=docs, show_progress=True)
             return nodes

dags/hivemind_mediawiki_etl.py

@@ -5,6 +5,7 @@
 
 from airflow import DAG
 from airflow.decorators import task
+from hivemind_etl_helpers.debugs import pring_qdrant_creds
 from hivemind_etl_helpers.mediawiki_etl import process_mediawiki_etl
 from hivemind_etl_helpers.src.utils.modules import ModulesMediaWiki
 
@@ -19,6 +20,8 @@ def get_mediawiki_communities() -> list[dict[str, str | list[str]]]:
         """
         Getting all communities having mediawiki from database
         """
+        # TODO: REMOVE!!!
+        pring_qdrant_creds()
         communities = ModulesMediaWiki().get_learning_platforms()
         return communities