Skip to content

Commit

Permalink
✨ !pic, !help, and gpt vision in thread chat
Browse files Browse the repository at this point in the history
  • Loading branch information
@hibobmaster
hibobmaster committed Apr 25, 2024
1 parent 81543d5 commit c5834db
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 16 deletions.
130 changes: 115 additions & 15 deletions src/bot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,33 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
image_url = (
f"data:{image_mimetype};base64,{b64_image}"
)
if (
"rel_type"
in event_source["content"]["m.relates_to"]
):
if (
"m.thread"
== event_source["content"]["m.relates_to"][
"rel_type"
]
):
thread_root_id = event_source["content"][
"m.relates_to"
]["event_id"]
asyncio.create_task(
self.gpt_vision_cmd(
room_id,
reply_to_event_id,
prompt,
image_url,
sender_id,
raw_user_message,
reply_in_thread=True,
thread_root_id=thread_root_id,
)
)
return

asyncio.create_task(
self.gpt_vision_cmd(
room_id,
Expand Down Expand Up @@ -408,20 +435,63 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
thread_root_id = event_source["content"]["m.relates_to"]["event_id"]
# thread is created by @bot
if thread_root_id in self.chatbot.conversation:
try:
asyncio.create_task(
self.thread_chat(
room_id,
reply_to_event_id,
sender_id=sender_id,
thread_root_id=thread_root_id,
prompt=content_body,
msgtype = event_source["content"]["msgtype"]
if "m.text" == msgtype:
# !pic command for thread chatting
p = self.pic_prog.search(content_body)
if p:
prompt = p.group(1)
try:
asyncio.create_task(
self.pic(
room_id,
prompt,
reply_to_event_id,
sender_id,
raw_user_message,
reply_in_thread=True,
thread_root_id=thread_root_id,
)
)
except Exception as e:
logger.error(e, exc_info=True)

return

# !help command for thread chatting
h = self.help_prog.search(content_body)
if h:
try:
asyncio.create_task(
self.help(
room_id,
reply_to_event_id,
sender_id,
raw_user_message,
reply_in_thread=True,
thread_root_id=thread_root_id,
)
)
except Exception as e:
logger.error(e, exc_info=True)

return

# normal chatting function
try:
asyncio.create_task(
self.thread_chat(
room_id,
reply_to_event_id,
sender_id=sender_id,
thread_root_id=thread_root_id,
prompt=content_body,
)
)
)
except Exception as e:
logger.error(e, exe_info=True)
except Exception as e:
logger.error(e, exe_info=True)

return
return

# common command

Expand Down Expand Up @@ -1508,6 +1578,8 @@ async def gpt_vision_cmd(
image_url: str,
sender_id: str,
user_message: str,
reply_in_thread=False,
thread_root_id=None,
) -> None:
try:
# sending typing state, seconds to milliseconds
Expand All @@ -1528,6 +1600,8 @@ async def gpt_vision_cmd(
reply_to_event_id=reply_to_event_id,
sender_id=sender_id,
user_message=user_message,
reply_in_thread=reply_in_thread,
thread_root_id=thread_root_id,
)
except Exception as e:
logger.error(e, exc_info=True)
Expand Down Expand Up @@ -1605,7 +1679,16 @@ async def new(
)

# !pic command
async def pic(self, room_id, prompt, replay_to_event_id, sender_id, user_message):
async def pic(
self,
room_id,
prompt,
replay_to_event_id,
sender_id,
user_message,
reply_in_thread=False,
thread_root_id=None,
):
try:
if self.image_generation_endpoint is not None:
await self.client.room_typing(room_id, timeout=int(self.timeout) * 1000)
Expand All @@ -1629,7 +1712,14 @@ async def pic(self, room_id, prompt, replay_to_event_id, sender_id, user_message
)
# send image
for image_path in image_path_list:
await send_room_image(self.client, room_id, image_path)
await send_room_image(
self.client,
room_id,
image_path,
replay_to_event_id,
reply_in_thread=reply_in_thread,
thread_root_id=thread_root_id,
)
await aiofiles.os.remove(image_path)
await self.client.room_typing(room_id, typing_state=False)
else:
Expand All @@ -1653,7 +1743,15 @@ async def pic(self, room_id, prompt, replay_to_event_id, sender_id, user_message
)

# !help command
async def help(self, room_id, reply_to_event_id, sender_id, user_message):
async def help(
self,
room_id,
reply_to_event_id,
sender_id,
user_message,
reply_in_thread=False,
thread_root_id=None,
):
help_info = (
"!gpt [prompt], generate a one time response without context conversation\n"
+ "!chat [prompt], chat with context conversation\n"
Expand All @@ -1672,6 +1770,8 @@ async def help(self, room_id, reply_to_event_id, sender_id, user_message):
sender_id=sender_id,
user_message=user_message,
reply_to_event_id=reply_to_event_id,
reply_in_thread=reply_in_thread,
thread_root_id=thread_root_id,
)

# send general error message
Expand Down
28 changes: 27 additions & 1 deletion src/send_image.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,14 @@
logger = getlogger()


async def send_room_image(client: AsyncClient, room_id: str, image: str):
async def send_room_image(
client: AsyncClient,
room_id: str,
image: str,
reply_to_event_id=None,
reply_in_thread=False,
thread_root_id=None,
):
"""
image: image path
"""
Expand Down Expand Up @@ -57,6 +64,25 @@ async def send_room_image(client: AsyncClient, room_id: str, image: str):
"url": resp.content_uri,
}

if reply_in_thread:
content = {
"body": os.path.basename(image), # descriptive title
"info": {
"size": file_stat.st_size,
"mimetype": mime_type,
"w": width, # width in pixel
"h": height, # height in pixel
},
"msgtype": "m.image",
"url": resp.content_uri,
"m.relates_to": {
"rel_type": "m.thread",
"event_id": thread_root_id,
"m.in_reply_to": {"event_id": reply_to_event_id},
"is_falling_back": True,
},
}

try:
await client.room_send(room_id, message_type="m.room.message", content=content)
except Exception as e:
Expand Down

0 comments on commit c5834db

Please sign in to comment.