Merge pull request #12 from TogetherCrew/feat/slash-commands

Feat: Integrate with discord slash commands

.github/workflows/production.yml

@@ -9,4 +9,4 @@ jobs:
   ci:
     uses: TogetherCrew/operations/.github/workflows/ci.yml@main
     secrets:
-      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

.github/workflows/start.staging.yml

@@ -6,4 +6,4 @@ jobs:
   ci:
     uses: TogetherCrew/operations/.github/workflows/ci.yml@main
     secrets:
-      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

.gitignore

@@ -160,4 +160,5 @@ cython_debug/
 #.idea/
 
 hivemind-bot-env/*
-main.ipynb
+main.ipynb
+.DS_Store

Dockerfile

@@ -1,5 +1,5 @@
 # It's recommended that we use `bullseye` for Python (alpine isn't suitable as it conflcts with numpy)
-FROM python:3.11-bullseye AS base 
+FROM python:3.11-bullseye AS base
 WORKDIR /project
 COPY . .
 RUN pip3 install -r requirements.txt

README.md

@@ -1,2 +1,3 @@
 # hivemind-bot
+
 This repository is made for TogetherCrew's LLM bot.

bot/retrievers/summary_retriever_base.py

@@ -42,7 +42,7 @@ def get_similar_nodes(
     ) -> list[NodeWithScore]:
         """
         get k similar nodes to the query.
-        Note: this funciton wold get the embedding
+        Note: this function wold get the embedding
         for the query to do the similarity search.
 
         Parameters

bot/retrievers/utils/load_hyperparams.py

@@ -14,7 +14,7 @@ def load_hyperparams() -> tuple[int, int, int]:
         to get the `k1` count similar nodes
     k2 : int
         the value for the secondary raw search
-        to get the `k2` count simliar nodes
+        to get the `k2` count similar nodes
     d : int
         the before and after day interval
     """

celery_app/tasks.py

@@ -1,29 +1,90 @@
-from celery_app.job_send import job_send
-from celery_app.server import app
-from utils.credentials import load_rabbitmq_credentials
+import json
+import logging
+from typing import Any
 
-# TODO: Write tasks that match our requirements
+from celery_app.server import app
+from celery_app.utils.fire_event import job_send
+from subquery import query_multiple_source
+from tc_messageBroker.rabbit_mq.event import Event
+from tc_messageBroker.rabbit_mq.payload.discord_bot.base_types.interaction_callback_data import (
+    InteractionCallbackData,
+)
+from tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction import (
+    ChatInputCommandInteraction,
+)
+from tc_messageBroker.rabbit_mq.payload.discord_bot.interaction_response import (
+    InteractionResponse,
+)
+from tc_messageBroker.rabbit_mq.payload.payload import Payload
+from tc_messageBroker.rabbit_mq.queue import Queue
 
 
 @app.task
-def add(x, y):
-    rabbit_creds = load_rabbitmq_credentials()
-    username = rabbit_creds["user"]
-    password = rabbit_creds["password"]
-    broker_url = rabbit_creds["host"]
-    port = rabbit_creds["port"]
+def ask_question_auto_search(
+    question: str,
+    community_id: str,
+    bot_given_info: ChatInputCommandInteraction,
+) -> None:
+    """
+    this task is for the case that the user asks a question
+    it would first retrieve the search metadata from summaries
+    then perform a query on the filetred raw data to find answer
 
-    res = x + y
-    job_send(broker_url, port, username, password, res)
+    Parameters
+    ------------
+    question : str
+        the user question
+    community_id : str
+        the community that the question was asked in
+    bot_given_info : tc_messageBroker.rabbit_mq.payload.discord_bot.chat_input_interaction.ChatInputCommandInteraction
+        the information data that needed to be sent back to the bot again.
+        This would be the `ChatInputCommandInteraction`.
+    """
+    prefix = f"COMMUNITY_ID: {community_id} | "
+    logging.info(f"{prefix}Processing question!")
+    create_interaction_content = Payload.DISCORD_BOT.INTERACTION_RESPONSE.Create(
+        interaction=bot_given_info.to_dict(),
+        data=InteractionResponse(
+            type=4,
+            data=InteractionCallbackData(
+                content="Processing your question ...", flags=64
+            ),
+        ),
+    ).to_dict()
 
-    return res
+    logging.info(f"{prefix}Sending process question to discord-bot!")
+    job_send(
+        event=Event.DISCORD_BOT.INTERACTION_RESPONSE.CREATE,
+        queue_name=Queue.DISCORD_BOT,
+        content=create_interaction_content,
+    )
+    logging.info(f"{prefix}Querying the data sources!")
+    # for now we have just the discord platform
+    response, source_nodes = query_multiple_source(
+        query=question,
+        community_id=community_id,
+        discord=True,
+    )
 
+    source_nodes_dict: list[dict[str, Any]] = []
+    for node in source_nodes:
+        node_dict = dict(node)
+        node_dict.pop("relationships", None)
+        source_nodes_dict.append(node_dict)
 
-@app.task
-def mul(x, y):
-    return x * y
+    results = {
+        "response": response,
+        "source_nodes": source_nodes_dict,
+    }
 
+    response_payload = Payload.DISCORD_BOT.INTERACTION_RESPONSE.Edit(
+        interaction=bot_given_info.to_dict(),
+        data=InteractionCallbackData(content=json.dumps(results)),
+    ).to_dict()
 
-@app.task
-def xsum(numbers):
-    return sum(numbers)
+    logging.info(f"{prefix}Sending Edit response to discord-bot!")
+    job_send(
+        event=Event.DISCORD_BOT.INTERACTION_RESPONSE.EDIT,
+        queue_name=Queue.DISCORD_BOT,
+        content=response_payload,
+    )

celery_app/utils/fire_event.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+from tc_messageBroker import RabbitMQ
+from utils.credentials import load_rabbitmq_credentials
+
+
+def job_send(event: str, queue_name: str, content: dict[str, Any]) -> None:
+    """
+    fire the data to a specific event on a specific queue
+
+    Parameters
+    -----------
+    event : str
+        the event to fire message to
+    queue_name : str
+        the queue to fire message on
+    content : dict[str, Any]
+        the content to send messages to
+    """
+
+    rabbit_creds = load_rabbitmq_credentials()
+    username = rabbit_creds["user"]
+    password = rabbit_creds["password"]
+    broker_url = rabbit_creds["host"]
+    port = rabbit_creds["port"]
+    rabbit_mq = RabbitMQ(
+        broker_url=broker_url, port=port, username=username, password=password
+    )
+    rabbit_mq.connect(queue_name)
+    rabbit_mq.publish(
+        queue_name=queue_name,
+        event=event,
+        content=content,
+    )

docker-compose.test.yml

@@ -49,7 +49,7 @@ services:
       - NEO4J_PLUGINS=["apoc", "graph-data-science"]
       - NEO4J_dbms_security_procedures_unrestricted=apoc.*,gds.*
     healthcheck:
-      test: ["CMD" ,"wget", "http://localhost:7474"]
+      test: ["CMD", "wget", "http://localhost:7474"]
       interval: 1m30s
       timeout: 10s
       retries: 2

docker-entrypoint.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 python3 -m coverage run --omit=tests/* -m pytest .
-python3 -m coverage lcov -o coverage/lcov.info
+python3 -m coverage lcov -o coverage/lcov.info

requirements.txt

@@ -1,5 +1,5 @@
 numpy
-llama-index>=0.9.26, <1.0.0
+llama-index>=0.9.27, <1.0.0
 pymongo
 python-dotenv
 pgvector
@@ -19,3 +19,4 @@ python-dotenv==1.0.0
 tc-hivemind-backend==1.1.0
 celery>=5.3.6, <6.0.0
 guidance
+tc-messageBroker>=1.6.4, <2.0.0

subquery.py

@@ -1,6 +1,6 @@
 from guidance.models import OpenAIChat
 from llama_index import QueryBundle, ServiceContext
-from llama_index.core import BaseQueryEngine
+from llama_index.core.base_query_engine import BaseQueryEngine
 from llama_index.query_engine import SubQuestionQueryEngine
 from llama_index.question_gen.guidance_generator import GuidanceQuestionGenerator
 from llama_index.schema import NodeWithScore
@@ -12,12 +12,12 @@
 def query_multiple_source(
     query: str,
     community_id: str,
-    discord: bool,
-    discourse: bool,
-    gdrive: bool,
-    notion: bool,
-    telegram: bool,
-    github: bool,
+    discord: bool = False,
+    discourse: bool = False,
+    gdrive: bool = False,
+    notion: bool = False,
+    telegram: bool = False,
+    github: bool = False,
 ) -> tuple[str, list[NodeWithScore]]:
     """
     query multiple platforms and get an answer from the multiple
@@ -30,23 +30,29 @@ def query_multiple_source(
         the community id to get their data
     discord : bool
         if `True` then add the engine to the subquery_generator
+        default is set to False
     discourse : bool
         if `True` then add the engine to the subquery_generator
+        default is set to False
     gdrive : bool
         if `True` then add the engine to the subquery_generator
+        default is set to False
     notion : bool
         if `True` then add the engine to the subquery_generator
+        default is set to False
     telegram : bool
         if `True` then add the engine to the subquery_generator
+        default is set to False
     github : bool
         if `True` then add the engine to the subquery_generator
+        default is set to False
 
 
     Returns
     --------
     response : str,
         the response to the user query from the LLM
-        using the engines of the given platforms (pltform equal to True)
+        using the engines of the given platforms (platform equal to True)
     source_nodes : list[NodeWithScore]
         the list of nodes that were source of answering
     """

tests/integration/test_fetch_community_id.py

@@ -0,0 +1,80 @@
+from datetime import datetime, timedelta
+from unittest import TestCase
+
+from bson import ObjectId
+from utils.fetch_community_id import fetch_community_id_by_guild_id
+from utils.mongo import MongoSingleton
+
+
+class TestFetchDiscordCommunityId(TestCase):
+    def add_platform(self):
+        client = MongoSingleton.get_instance().get_client()
+
+        action = {
+            "INT_THR": 1,
+            "UW_DEG_THR": 1,
+            "PAUSED_T_THR": 1,
+            "CON_T_THR": 4,
+            "CON_O_THR": 3,
+            "EDGE_STR_THR": 5,
+            "UW_THR_DEG_THR": 5,
+            "VITAL_T_THR": 4,
+            "VITAL_O_THR": 3,
+            "STILL_T_THR": 2,
+            "STILL_O_THR": 2,
+            "DROP_H_THR": 2,
+            "DROP_I_THR": 1,
+        }
+
+        client["Core"]["platforms"].insert_one(
+            {
+                "_id": ObjectId(self.platform_id),
+                "name": "discord",
+                "metadata": {
+                    "id": self.guild_id,
+                    "icon": "111111111111111111111111",
+                    "name": "A guild",
+                    "selectedChannels": [
+                        {"channelId": "1020707129214111827", "channelName": "general"}
+                    ],
+                    "window": {"period_size": 7, "step_size": 1},
+                    "action": action,
+                    "period": datetime.now() - timedelta(days=30),
+                },
+                "community": ObjectId(self.community_id),
+                "disconnectedAt": None,
+                "connectedAt": (datetime.now() - timedelta(days=40)),
+                "isInProgress": True,
+                "createdAt": datetime(2023, 11, 1),
+                "updatedAt": datetime(2023, 11, 1),
+            }
+        )
+
+    def delete_platform(self):
+        client = MongoSingleton.get_instance().get_client()
+        client["Core"]["platforms"].delete_one({"_id": ObjectId(self.platform_id)})
+
+    def test_get_guild_id(self):
+        self.platform_id = "515151515151515151515151"
+        self.guild_id = "1234"
+        self.community_id = "aabbccddeeff001122334455"
+        self.delete_platform()
+        self.add_platform()
+
+        community_id = fetch_community_id_by_guild_id(guild_id=self.guild_id)
+
+        self.assertEqual(community_id, self.community_id)
+
+    def test_get_guild_id_no_data(self):
+        self.platform_id = "515151515151515151515151"
+        self.guild_id = "1234"
+        self.community_id = "aabbccddeeff001122334455"
+
+        self.delete_platform()
+        self.add_platform()
+
+        client = MongoSingleton.get_instance().get_client()
+        client["Core"]["platforms"].delete_one({"_id": ObjectId(self.platform_id)})
+
+        with self.assertRaises(ValueError):
+            _ = fetch_community_id_by_guild_id(guild_id=self.guild_id)