Merge pull request #26 from TogetherCrew/feat/retry-rmq-connection

feat: retrying rmq connection in case of connection error!

requirements.txt

@@ -20,4 +20,5 @@ tc-hivemind-backend==1.1.2
 celery>=5.3.6, <6.0.0
 guidance
 tc-messageBroker==1.6.6
-traceloop-sdk==0.9.4
+traceloop-sdk==0.9.4
+backoff==2.2.1

worker.py

@@ -2,7 +2,9 @@
 import logging
 from typing import Any
 
+import backoff
 from celery_app.tasks import ask_question_auto_search
+from pika.exceptions import ConnectionClosedByBroker
 from tc_messageBroker import RabbitMQ
 from tc_messageBroker.rabbit_mq.event import Event
 from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import (
@@ -38,6 +40,12 @@ def query_llm(recieved_data: dict[str, Any]):
     )
 
 
+@backoff.on_exception(
+    wait_gen=backoff.expo,
+    exception=(ConnectionClosedByBroker, ConnectionError),
+    # waiting for 3 hours
+    max_time=60 * 60 * 3,
+)
 def job_recieve(broker_url, port, username, password):
     rabbit_mq = RabbitMQ(
         broker_url=broker_url, port=port, username=username, password=password
@@ -50,7 +58,7 @@ def job_recieve(broker_url, port, username, password):
     if rabbit_mq.channel is not None:
         rabbit_mq.channel.start_consuming()
     else:
-        print("Connection to broker was not successful!")
+        logging.error("Connection to broker was not successful!")
 
 
 if __name__ == "__main__":