From 53ef70ef1869fd81e67ef4a5fe639d88acbc74e4 Mon Sep 17 00:00:00 2001 From: Mateusz Baginski Date: Thu, 26 Dec 2024 13:38:29 +0100 Subject: [PATCH] feat(chat): better file attachement handling --- .../src/modules/messages/messages.service.ts | 26 ++++++++++++++++++- .../create-relevant-embeddings-prompt.ts | 21 +++++++++++---- .../input-toolbar/chat-input-toolbar.tsx | 2 +- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/apps/backend/src/modules/messages/messages.service.ts b/apps/backend/src/modules/messages/messages.service.ts index 69691877..774a699d 100644 --- a/apps/backend/src/modules/messages/messages.service.ts +++ b/apps/backend/src/modules/messages/messages.service.ts @@ -4,7 +4,7 @@ import { taskEither as TE } from 'fp-ts'; import { pipe } from 'fp-ts/lib/function'; import { delay, inject, injectable } from 'tsyringe'; -import { findItemIndexById, mapAsyncIterator, tryOrThrowTE } from '@llm/commons'; +import { findItemIndexById, mapAsyncIterator, pluckTyped, tryOrThrowTE } from '@llm/commons'; import { groupSdkAIMessagesByRepeats, type SdkCreateMessageInputT, @@ -85,7 +85,31 @@ export class MessagesService implements WithAuthFirewall { messageId: id, ...file, })), + TE.chainW(files => this.projectsFilesService.search({ + sort: 'createdAt:desc', + limit: files.length, + projectId: project.id, + offset: 0, + ids: pipe([...files], pluckTyped('id')), + })), )), + TE.chainW(attachedFiles => this.repo.create({ + value: { + chatId: chat.id, + metadata: {}, + aiModelId: null, + creatorUserId: creator.id, + role: 'system', + content: [ + 'User attached to chat these files:', + ...attachedFiles.items.map(({ description, resource }) => + `- ${resource.name} - ${description}`, + ), + '---', + 'If there is application attached to this chat, process these files with it.', + ].join('\n'), + }, + })), ); }), TE.tap(({ id }) => this.esIndexRepo.findAndIndexDocumentById(id)), diff --git a/apps/backend/src/modules/projects-embeddings/helpers/create-relevant-embeddings-prompt.ts b/apps/backend/src/modules/projects-embeddings/helpers/create-relevant-embeddings-prompt.ts index 3be77bd8..67e2e135 100644 --- a/apps/backend/src/modules/projects-embeddings/helpers/create-relevant-embeddings-prompt.ts +++ b/apps/backend/src/modules/projects-embeddings/helpers/create-relevant-embeddings-prompt.ts @@ -2,7 +2,10 @@ import type { EsMatchingProjectEmbedding } from '../elasticsearch'; import { groupEmbeddingsByFile } from './group-embeddings'; -export function createRelevantEmbeddingsPrompt(message: string, embeddings: EsMatchingProjectEmbedding[]): string { +export function createRelevantEmbeddingsPrompt( + message: string, + embeddings: EsMatchingProjectEmbedding[], +): string { const groupedEmbeddings = groupEmbeddingsByFile(embeddings); const fragmentsText = Object @@ -32,10 +35,15 @@ export function createRelevantEmbeddingsPrompt(message: string, embeddings: EsMa '', 'Core Instructions:', '1. Respond in the same language as the user\'s message', - '2. Consider and analyze ALL provided file fragments', - '3. Search through ALL file types, including source code, documents, images (.png, .jpg, etc), presentations, and any other files', - '4. MANDATORY: Never mention filenames directly in text - always use #embedding: instead', - '5. MANDATORY: Always consider image files in your search - they may contain crucial diagrams, screenshots or visual documentation', + '2. If message contains #app mention: MAINTAIN THE APP\'S PERSONALITY AND TONE throughout the response', + '3. If responding as an app: DO NOT describe files, instead keep acting as the app would', + '4. Consider and analyze ALL provided file fragments', + '5. Search through ALL file types, including source code, documents, images (.png, .jpg, etc), presentations, and any other files', + '6. MANDATORY: Never mention filenames directly in text - always use #embedding: instead', + '7. MANDATORY: Always consider image files in your search - they may contain crucial diagrams, screenshots or visual documentation', + '8. MANDATORY: If files were attached to the chat or previous messages - treat them with highest priority', + '9. MANDATORY: Always analyze attached files first before other context', + '10. MANDATORY: For attached files, provide more detailed analysis unless user specifies otherwise', '', 'Text Format Rules:', '- ❌ WRONG: "In config.ts we see..."', @@ -66,6 +74,9 @@ export function createRelevantEmbeddingsPrompt(message: string, embeddings: EsMa '- For file queries: Provide file name and one-sentence description unless more details requested', '- If relevant file found: Suggest its potential usefulness', '- If context not relevant: Provide general response', + '- If user uploads a file without asking about it: DO NOT comment on or describe the file', + '- If files were attached: Give them priority in analysis and responses', + '- For attached files: Provide more detailed insights unless explicitly asked not to', '- When mentioning files, add RELEVANT action buttons based on context:', ' CRITICAL: Button labels and actions:', ' - MUST be in user\'s language', diff --git a/apps/chat/src/modules/chats/conversation/input-toolbar/chat-input-toolbar.tsx b/apps/chat/src/modules/chats/conversation/input-toolbar/chat-input-toolbar.tsx index 3fc4fe82..743e6753 100644 --- a/apps/chat/src/modules/chats/conversation/input-toolbar/chat-input-toolbar.tsx +++ b/apps/chat/src/modules/chats/conversation/input-toolbar/chat-input-toolbar.tsx @@ -161,7 +161,7 @@ export function ChatInputToolbar(