feat: added LLM traces feature!

dags/hivemind_etl.py

@@ -32,6 +32,7 @@
 from hivemind_etl_helpers.src.utils.get_mongo_discord_communities import (
     get_all_discord_communities,
 )
+from traceloop.sdk import Traceloop
 
 # def setup_phonix():
 #     _ = px.launch_app()
@@ -54,6 +55,7 @@ def get_discord_communities() -> list[str]:
         """
         Getting all communities having discord from database
         """
+        setup_traceloop()
         communities = get_all_discord_communities()
         return communities
 
@@ -82,6 +84,7 @@ def get_mongo_discord_communities() -> list[str]:
         this function is the same with `get_discord_communities`
         we just changed the name for the pylint
         """
+        setup_traceloop()
         communities = get_all_discord_communities()
         return communities
 
@@ -94,3 +97,7 @@ def start_discord_summary_vectorstore(community_id: str):
 
     communities = get_mongo_discord_communities()
     start_discord_summary_vectorstore.expand(community_id=communities)
+
+
+def setup_traceloop():
+    Traceloop.init()

docker-compose.test.yml

@@ -26,6 +26,7 @@ services:
       - EMBEDDING_DIM=1024
       - OPENAI_API_KEY=some_credentials
       - COHERE_API_KEY=some_credentials
+      - TRACELOOP_BASE_URL=some_url
     volumes:
       - ./coverage:/project/coverage
     depends_on:

requirements.txt

@@ -17,3 +17,4 @@ coverage>=7.3.3, <8.0.0
 pytest>=7.4.3, <8.0.0
 python-dotenv>=1.0.0, <2.0.0
 tc-hivemind-backend==1.1.1
+traceloop-sdk