refactor: supports image message (#190)

close #184
petercat-ai · Aug 7, 2024 · ed1e193 · ed1e193
2 parents 81f3d5a + 941fabc
commit ed1e193
Show file tree

Hide file tree

Showing 14 changed files with 3,882 additions and 29,666 deletions.
diff --git a/client/package-lock.json b/client/package-lock.json
diff --git a/client/package.json b/client/package.json
@@ -39,7 +39,7 @@
     "lottie-react": "^2.4.0",
     "next": "14.0.1",
     "openai": "^4.24.7",
-    "petercat-lui": "^0.0.24",
+    "petercat-lui": "^0.1.1",
     "postcss": "8.4.27",
     "react": "18.2.0",
     "react-dom": "18.2.0",

diff --git a/client/yarn.lock b/client/yarn.lock
diff --git a/extension/package.json b/extension/package.json
@@ -1,6 +1,6 @@
 {
   "name": "extension",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -15,7 +15,7 @@
     "mini-css-extract-plugin": "^2.9.0",
     "next": "14.2.5",
     "next-compose-plugins": "^2.2.1",
-    "petercat-lui": "^0.0.24",
+    "petercat-lui": "^0.1.0",
     "postcss-loader": "^8.1.1",
     "react": "^18",
     "react-dom": "^18",

diff --git a/lui/package.json b/lui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "petercat-lui",
-  "version": "0.0.24",
+  "version": "0.1.1",
   "description": "A react library developed with dumi",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",

diff --git a/lui/src/Chat/index.tsx b/lui/src/Chat/index.tsx
@@ -19,7 +19,7 @@ import useSWR from 'swr';
 import StopBtn from '../StopBtn';
 import ThoughtChain from '../ThoughtChain';
 import SignatureIcon from '../icons/SignatureIcon';
-import { Role } from '../interface';
+import { Message, Role } from '../interface';
 import { BOT_INFO } from '../mock';
 import { fetcher, streamChat } from '../services/ChatController';
 import { convertChunkToJson, handleStream } from '../utils';
@@ -134,9 +134,6 @@ const Chat: FC<ChatProps> = memo(
             chatRef={proChatRef}
             helloMessage={botInfo.helloMessage}
             userMeta={{ title: 'User' }}
-            backToBottomConfig={{
-              style: { display: 'none' },
-            }}
             chatItemRenderConfig={{
               avatarRender: (props: ChatItemProps) => {
                 if (props.originData?.role === Role.user) {
@@ -154,6 +151,7 @@ const Chat: FC<ChatProps> = memo(
                 if (originData?.role === Role.user) {
                   return defaultDom;
                 }
+                console.log('originData', originData);
 
                 const originMessage = convertChunkToJson(
                   originData.content,
@@ -240,10 +238,13 @@ const Chat: FC<ChatProps> = memo(
                 )
                 .map((message) => ({
                   role: message.role,
-                  content: JSON.stringify(
-                    convertChunkToJson(message.content as string),
-                  ),
-                }));
+                  content: [
+                    {
+                      type: 'text',
+                      text: message.content,
+                    },
+                  ],
+                })) as Message[];
 
               const response = await streamChat(
                 newMessages,

diff --git a/lui/src/Chat/inputArea/InputArea.tsx b/lui/src/Chat/inputArea/InputArea.tsx
@@ -1,15 +1,15 @@
 import { Button, Form, Input } from 'antd';
 import React, { useState } from 'react';
-import { SendMessageIcon } from "../../icons/SendMessageIcon";
-import { StopMessageIcon } from "../../icons/StopMessageIcon";
-import { NewMessageIcon } from "../../icons/NewMessageIcon";
-import { UploadImageIcon } from "../../icons/UploadImageIcon";
+import { NewMessageIcon } from '../../icons/NewMessageIcon';
+import { SendMessageIcon } from '../../icons/SendMessageIcon';
+import { StopMessageIcon } from '../../icons/StopMessageIcon';
+import { UploadImageIcon } from '../../icons/UploadImageIcon';
 
 const InputAreaRender = (props: {
-  isShowStop: boolean,
-  onMessageSend: (message: string) => void | Promise<any>,
-  onClear: () => void,
-  onStop: () => void,
+  isShowStop: boolean;
+  onMessageSend: (message: string) => void | Promise<any>;
+  onClear: () => void;
+  onStop: () => void;
 }) => {
   const [form] = Form.useForm();
   const [message, setMessage] = useState('');
@@ -19,9 +19,13 @@ const InputAreaRender = (props: {
       form.resetFields();
       props.onMessageSend(value.question ?? '');
     }
-  }
+  };
   const handleKeyDown = (event: React.KeyboardEvent<HTMLTextAreaElement>) => {
-    if (event.key === 'Enter' && !event.shiftKey) {
+    if (
+      event.key === 'Enter' &&
+      !event.shiftKey &&
+      !event.nativeEvent.isComposing
+    ) {
       setMessage('');
       form.resetFields();
       props.onMessageSend(message);
@@ -34,40 +38,59 @@ const InputAreaRender = (props: {
     <Form
       form={form}
       onFinish={finish}
-      className='px-[12px] py-[10px] m-[12px] rounded-[10px] lui-input-area bg-[#f1f1f1]'
+      className="px-[12px] py-[10px] m-[12px] rounded-[10px] lui-input-area bg-[#f1f1f1]"
     >
-      <Form.Item
-        name="question"
-      >
-        <Input.TextArea value={message} onChange={handleChange} style={{ height: 100, border: 'none', resize: 'none', backgroundColor: '#F1F1F1' }} onKeyDown={handleKeyDown} />
+      <Form.Item name="question">
+        <Input.TextArea
+          value={message}
+          onChange={handleChange}
+          style={{
+            height: 100,
+            border: 'none',
+            resize: 'none',
+            backgroundColor: '#F1F1F1',
+          }}
+          onKeyDown={handleKeyDown}
+        />
       </Form.Item>
       <div className="flex w-[100%]">
         <div className="space-x-2 flex-1">
-          {
-            props && props.isShowStop && (
-              <Button type="primary" className="bg-white hover:!bg-white" icon={<StopMessageIcon />}
-                onClick={() => {
-                  if (props && props.onStop) {
-                    props.onStop();
-                  }
-                }}
-              >
-              </Button>
-            )
-          }
-          <Button type="primary" className="bg-white hover:!bg-white" onClick={() => {
-            if (props && props.onClear) {
-              props.onClear();
-            }
-          }} icon={<NewMessageIcon />}>
-          </Button>
-          <Button type="primary" className="bg-white hover:!bg-white" icon={<UploadImageIcon />}>
-          </Button>
+          {props && props.isShowStop && (
+            <Button
+              type="primary"
+              className="bg-white hover:!bg-white"
+              icon={<StopMessageIcon />}
+              onClick={() => {
+                if (props && props.onStop) {
+                  props.onStop();
+                }
+              }}
+            ></Button>
+          )}
+          <Button
+            type="primary"
+            className="bg-white hover:!bg-white"
+            onClick={() => {
+              if (props && props.onClear) {
+                props.onClear();
+              }
+            }}
+            icon={<NewMessageIcon />}
+          ></Button>
+          <Button
+            type="primary"
+            className="bg-white hover:!bg-white"
+            icon={<UploadImageIcon />}
+          ></Button>
         </div>
-        <Button type="primary" className="w-[32px] bg-gray-700 hover:!bg-gray-700" htmlType="submit" icon={<SendMessageIcon />}>
-        </Button>
+        <Button
+          type="primary"
+          className="w-[32px] bg-gray-700 hover:!bg-gray-700"
+          htmlType="submit"
+          icon={<SendMessageIcon />}
+        ></Button>
       </div>
     </Form>
   );
 };
-export default InputAreaRender;
+export default InputAreaRender;
diff --git a/lui/src/interface/contentMessage.interface.ts b/lui/src/interface/contentMessage.interface.ts
@@ -36,9 +36,26 @@ export interface IParticipant {
   avatar: string;
 }
 
-export interface IPrompt {
-  content: string;
+interface ImageURL {
+  url: string;
+  detail?: 'auto' | 'low' | 'high';
+}
+
+interface ImageURLContentBlock {
+  image_url: ImageURL;
+  type: 'image_url';
+}
+
+interface TextContentBlock {
+  text: string;
+  type: 'text';
+}
+
+type MessageContent = ImageURLContentBlock | TextContentBlock;
+
+export interface Message {
   role: string;
+  content: MessageContent[];
 }
 
 export interface IMessage {

diff --git a/lui/src/services/ChatController.ts b/lui/src/services/ChatController.ts
@@ -1,12 +1,12 @@
 import axios from 'axios';
-import { IPrompt } from '../interface';
+import { Message } from '../interface';
 
 /**
  * Chat api
- * @param message IPrompt
+ * @param message
  */
 export async function streamChat(
-  messages: IPrompt[],
+  messages: Message[],
   apiDomain: string,
   apiUrl = '/api/chat/stream_qa',
   prompt = '',

diff --git a/lui/src/utils/chatTranslator.ts b/lui/src/utils/chatTranslator.ts
@@ -16,11 +16,17 @@ export const convertChunkToJson = (rawData: string) => {
 
   try {
     forEach(chunks, (chunk) => {
-      const parsedChunk = JSON.parse(chunk);
-      if (parsedChunk.type === 'tool') {
-        tools.push(parsedChunk);
-      } else if (parsedChunk.type === 'message') {
-        messages.push(parsedChunk.content);
+      const regex = /data: (.*?})\s*$/;
+      const match = chunk.match(regex);
+      if (match && match[1]) {
+        const parsedChunk = JSON.parse(match[1]);
+        if (parsedChunk.type === 'tool') {
+          tools.push(parsedChunk);
+        } else if (parsedChunk.type === 'message') {
+          messages.push(parsedChunk.content);
+        }
+      } else {
+        messages.push(chunk);
       }
     });
     return { tools, message: messages.join('') };

diff --git a/server/agent/base.py b/server/agent/base.py
@@ -1,6 +1,5 @@
 import json
 from typing import AsyncIterator, Dict, Callable, Optional
-# import uuid
 from langchain.agents import AgentExecutor
 from data_class import ChatData, Message
 from langchain.agents.format_scratchpad.openai_tools import (
@@ -107,8 +106,6 @@ def chat_history_transform(messages: list[Message]):
     async def run_stream_chat(self, input_data: ChatData) -> AsyncIterator[str]:
         try:
             messages = input_data.messages
-            print(self.chat_history_transform(messages))
-
             async for event in self.agent_executor.astream_events(
                 {
                     "input": messages[len(messages) - 1].content,
@@ -117,34 +114,20 @@ async def run_stream_chat(self, input_data: ChatData) -> AsyncIterator[str]:
                 version="v1",
             ):
                 kind = event["event"]
-                if kind == "on_chain_start":
-                    if (
-                        event["name"] == "agent"
-                    ): 
-                        print(
-                        f"Starting agent: {event['name']} "
-                        f"with input: {event['data'].get('input')}"
-                        )
-                elif kind == "on_chain_end":
-                    if (
-                        event["name"] == "agent"
-                    ): 
-                        print (
-                            f"Done agent: {event['name']} "
-                            f"with output: {event['data'].get('output')['output']}"
-                        )
-                if kind == "on_chat_model_stream":
-                    # id =  str(uuid.uuid4())
+                print("event", kind, event)
+                if kind == "on_llm_stream" or kind == "on_chat_model_stream":
                     content = event["data"]["chunk"].content
                     if content:
                         json_output = json.dumps({
+                            "id": event["run_id"],
                             "type": "message",
                             "content": content,
                         }, ensure_ascii=False)
-                        yield f"{json_output}\n\n"
+                        yield f"data: {json_output}\n\n"
                 elif kind == "on_tool_start":
                     children_value = event["data"].get("input", {})
                     json_output = json.dumps({
+                        "id": event["run_id"],
                         "type": "tool",
                         "extra": {
                             "source": f"已调用工具: {event['name']}",
@@ -154,10 +137,11 @@ async def run_stream_chat(self, input_data: ChatData) -> AsyncIterator[str]:
                         }
                     }, ensure_ascii=False)
 
-                    yield f"{json_output}\n\n"
+                    yield f"data: {json_output}\n\n"
                 elif kind == "on_tool_end":
                     children_value = event["data"].get("output", {})
                     json_output = json.dumps({
+                        "id": event["run_id"],
                         "type": "tool",
                         "extra": {
                             "source": f"已调用工具: {event['name']}",
@@ -166,15 +150,17 @@ async def run_stream_chat(self, input_data: ChatData) -> AsyncIterator[str]:
                             "status": "success"
                         },
                     }, ensure_ascii=False)
-                    yield f"{json_output}\n\n"
+                    yield f"data: {json_output}\n\n"
         except Exception as e:
-            yield f"error: {str(e)}\n\n"
+            res = {
+                "status": "error",
+                "message": str(e)
+            }
+            yield f"data: {json.dumps(res, ensure_ascii=False)}\n\n"
 
     async def run_chat(self, input_data: ChatData) -> str:
         try:
-            messages = input_data.messages
-            print('history', self.chat_history_transform(messages))
-
+            messages = input_data.messages        
             return self.agent_executor.invoke(
                 {
                     "input": messages[len(messages) - 1].content,