From 9b53fb6bb729cf546d519005a014f44d438959d6 Mon Sep 17 00:00:00 2001 From: Mateusz Baginski Date: Wed, 25 Dec 2024 13:17:22 +0100 Subject: [PATCH 01/10] feat(backend): add internal projects support --- .../0025-add-internal-projects-fields.ts | 31 +++++++++++++++++++ apps/backend/src/migrations/index.ts | 2 ++ .../projects-files/projects-files.repo.ts | 8 ++++- .../projects-files/projects-files.tables.ts | 6 +++- .../elasticsearch/projects-es-index.repo.ts | 3 ++ .../elasticsearch/projects-es-search.repo.ts | 10 ++++-- .../src/modules/projects/projects.service.ts | 3 +- .../src/modules/projects/projects.tables.ts | 1 + 8 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 apps/backend/src/migrations/0025-add-internal-projects-fields.ts diff --git a/apps/backend/src/migrations/0025-add-internal-projects-fields.ts b/apps/backend/src/migrations/0025-add-internal-projects-fields.ts new file mode 100644 index 00000000..5580dc03 --- /dev/null +++ b/apps/backend/src/migrations/0025-add-internal-projects-fields.ts @@ -0,0 +1,31 @@ +import type { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable('projects_files') + .addColumn('message_id', 'uuid', col => col.references('messages.id').onDelete('restrict')) + .execute(); + + await db.schema + .createIndex('projects_files_message_id_index') + .on('projects_files') + .column('message_id') + .execute(); + + await db.schema + .alterTable('projects') + .addColumn('internal', 'boolean', col => col.notNull().defaultTo(false)) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable('projects') + .dropColumn('internal') + .execute(); + + await db.schema + .alterTable('projects_files') + .dropColumn('message_id') + .execute(); +} diff --git a/apps/backend/src/migrations/index.ts b/apps/backend/src/migrations/index.ts index 68b07027..c2071745 100644 --- a/apps/backend/src/migrations/index.ts +++ b/apps/backend/src/migrations/index.ts @@ -23,6 +23,7 @@ import * as dropUniqueNameFromS3Assets from './0021-drop-unique-name-from-s3-ass import * as addIdToProjectFilesTable from './0022-add-id-to-project-files-table'; import * as addProjectsEmbeddingsTable from './0023-add-projects-embeddings-table'; import * as dropUnusedImagesTable from './0024-drop-unused-images-table'; +import * as addInternalProjectsFields from './0025-add-internal-projects-fields'; export const DB_MIGRATIONS = { '0000-add-users-tables': addUsersTables, @@ -50,4 +51,5 @@ export const DB_MIGRATIONS = { '0022-add-id-to-project-files-table': addIdToProjectFilesTable, '0023-add-projects-embeddings-table': addProjectsEmbeddingsTable, '0024-drop-unused-images-table': dropUnusedImagesTable, + '0025-add-internal-projects-fields': addInternalProjectsFields, }; diff --git a/apps/backend/src/modules/projects-files/projects-files.repo.ts b/apps/backend/src/modules/projects-files/projects-files.repo.ts index 789e4dd0..2044bcbc 100644 --- a/apps/backend/src/modules/projects-files/projects-files.repo.ts +++ b/apps/backend/src/modules/projects-files/projects-files.repo.ts @@ -97,6 +97,8 @@ export class ProjectsFilesRepo extends createDatabaseRepo('projects_files') { s3_resource_created_at: s3ResourceCreatedAt, s3_resource_updated_at: s3ResourceUpdatedAt, + message_id: messageId, + ...item }): ProjectFileTableRowWithRelations => ({ ...camelcaseKeys(item), @@ -114,11 +116,15 @@ export class ProjectsFilesRepo extends createDatabaseRepo('projects_files') { name: bucketName, }, }, - project: { id: projectId, name: projectName, }, + message: messageId + ? { + id: messageId, + } + : null, })), ), ); diff --git a/apps/backend/src/modules/projects-files/projects-files.tables.ts b/apps/backend/src/modules/projects-files/projects-files.tables.ts index 167ffcae..c9799eb5 100644 --- a/apps/backend/src/modules/projects-files/projects-files.tables.ts +++ b/apps/backend/src/modules/projects-files/projects-files.tables.ts @@ -5,6 +5,8 @@ import type { NormalizeSelectTableRow, TableId, TableRowWithIdName, + TableRowWithUuid, + TableUuid, TableWithDefaultColumns, } from '../database'; import type { S3ResourcesTableRowWithRelations } from '../s3'; @@ -14,6 +16,7 @@ export type ProjectsFilesTable = & { project_id: ColumnType; s3_resource_id: ColumnType; + message_id: ColumnType; }; export type ProjectFileTableRow = NormalizeSelectTableRow; @@ -21,9 +24,10 @@ export type ProjectFileTableRow = NormalizeSelectTableRow; export type ProjectFileTableInsertRow = NormalizeInsertTableRow; export type ProjectFileTableRowWithRelations = - & Omit + & Omit & { resource: S3ResourcesTableRowWithRelations; project: TableRowWithIdName; + message: TableRowWithUuid | null; description: string | null; }; diff --git a/apps/backend/src/modules/projects/elasticsearch/projects-es-index.repo.ts b/apps/backend/src/modules/projects/elasticsearch/projects-es-index.repo.ts index 0c99438c..17f7e068 100644 --- a/apps/backend/src/modules/projects/elasticsearch/projects-es-index.repo.ts +++ b/apps/backend/src/modules/projects/elasticsearch/projects-es-index.repo.ts @@ -29,6 +29,9 @@ const ProjectsAbstractEsIndexRepo = createElasticsearchIndexRepo({ ...createBaseAutocompleteFieldMappings(), ...createArchivedRecordMappings(), organization: createIdNameObjectMapping(), + internal: { + type: 'keyword', + }, description: { type: 'text', analyzer: 'folded_lowercase_analyzer', diff --git a/apps/backend/src/modules/projects/elasticsearch/projects-es-search.repo.ts b/apps/backend/src/modules/projects/elasticsearch/projects-es-search.repo.ts index 89d29821..591f6825 100644 --- a/apps/backend/src/modules/projects/elasticsearch/projects-es-search.repo.ts +++ b/apps/backend/src/modules/projects/elasticsearch/projects-es-search.repo.ts @@ -20,6 +20,10 @@ import { ProjectsEsIndexRepo, } from './projects-es-index.repo'; +type InternalSearchProjectsInputT = SdKSearchProjectsInputT & { + excludeInternal?: boolean; +}; + @injectable() export class ProjectsEsSearchRepo { constructor( @@ -31,7 +35,7 @@ export class ProjectsEsSearchRepo { TE.map(ProjectsEsSearchRepo.mapOutputHit), ); - search = (dto: SdKSearchProjectsInputT) => + search = (dto: InternalSearchProjectsInputT) => pipe( this.indexRepo.search( ProjectsEsSearchRepo.createEsRequestSearchBody(dto).toJSON(), @@ -53,11 +57,12 @@ export class ProjectsEsSearchRepo { private static createEsRequestSearchFilters = ( { + excludeInternal = true, phrase, ids, organizationIds, archived, - }: SdKSearchProjectsInputT, + }: InternalSearchProjectsInputT, ): esb.Query => esb.boolQuery().must( rejectFalsyItems([ @@ -73,6 +78,7 @@ export class ProjectsEsSearchRepo { .minimumShouldMatch(1) ), !isNil(archived) && esb.termQuery('archived', archived), + excludeInternal && esb.termQuery('internal', false), ]), ); diff --git a/apps/backend/src/modules/projects/projects.service.ts b/apps/backend/src/modules/projects/projects.service.ts index 6d4d15c9..84742da4 100644 --- a/apps/backend/src/modules/projects/projects.service.ts +++ b/apps/backend/src/modules/projects/projects.service.ts @@ -76,10 +76,11 @@ export class ProjectsService implements WithAuthFirewall { search = this.esSearchRepo.search; - create = ({ organization, ...values }: SdkCreateProjectInputT) => pipe( + create = ({ internal, organization, ...values }: SdkCreateProjectInputT & { internal?: boolean; }) => pipe( this.repo.create({ value: { ...values, + internal: !!internal, organizationId: organization.id, }, }), diff --git a/apps/backend/src/modules/projects/projects.tables.ts b/apps/backend/src/modules/projects/projects.tables.ts index e7e4d1d9..38042cc5 100644 --- a/apps/backend/src/modules/projects/projects.tables.ts +++ b/apps/backend/src/modules/projects/projects.tables.ts @@ -15,6 +15,7 @@ export type ProjectsTable = organization_id: ColumnType; name: string; description: string | null; + internal: boolean; }; export type ProjectTableRow = NormalizeSelectTableRow; From 503111df1bfe648194961c89697ab74635732f59 Mon Sep 17 00:00:00 2001 From: Mateusz Baginski Date: Wed, 25 Dec 2024 13:29:25 +0100 Subject: [PATCH 02/10] feat(sdk): add `files` to `SdkCreateMessageInputT` --- .../src/modules/dashboard/messages/dto/sdk-create-message.dto.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sdk/src/modules/dashboard/messages/dto/sdk-create-message.dto.ts b/packages/sdk/src/modules/dashboard/messages/dto/sdk-create-message.dto.ts index e0b81da9..b185dbe8 100644 --- a/packages/sdk/src/modules/dashboard/messages/dto/sdk-create-message.dto.ts +++ b/packages/sdk/src/modules/dashboard/messages/dto/sdk-create-message.dto.ts @@ -5,6 +5,7 @@ import { SdkTableRowWithUuidV } from '~/shared'; export const SdkCreateMessageInputV = z.object({ content: z.string(), replyToMessage: SdkTableRowWithUuidV.optional().nullable(), + files: z.array(z.instanceof(File)).optional().nullable(), }); export type SdkCreateMessageInputT = z.infer; From 0ee8f02bef7bcb84effdaa46091ce1aef31941da Mon Sep 17 00:00:00 2001 From: Mateusz Baginski Date: Wed, 25 Dec 2024 16:05:07 +0100 Subject: [PATCH 03/10] feat(chat): add `files` cards list --- .../projects-embeddings.service.ts | 16 +-- .../conversation/chat-conversation-panel.tsx | 6 +- .../chats/conversation/files/file-card.tsx | 122 ++++++++++++++++++ .../conversation/files/files-cards-list.tsx | 59 +++++++++ .../modules/chats/conversation/files/index.ts | 3 + .../conversation/files/select-chat-file.tsx | 3 + .../src/modules/chats/conversation/index.ts | 9 ++ .../chats/start-chat/start-chat-form.tsx | 40 ++++-- .../chats/start-chat/use-start-chat-form.tsx | 1 + .../modules/projects/files/use-file-upload.ts | 4 +- packages/commons/src/helpers/index.ts | 1 + .../commons/src/helpers}/mimetypes/index.ts | 0 .../helpers}/mimetypes/is-image-mimetype.ts | 0 .../mimetypes/is-legacy-excel-mimetype.ts | 0 .../mimetypes/is-legacy-word-mimetype.ts | 0 .../src/helpers}/mimetypes/is-pdf-mimetype.ts | 0 .../mimetypes/is-xml-office-mimetype.ts | 0 packages/commons/src/helpers/urls/index.ts | 2 + .../src/helpers/urls/is-pdf-file-url.ts | 3 + .../src/helpers/urls/is-word-file-url.ts | 3 + 20 files changed, 249 insertions(+), 23 deletions(-) create mode 100644 apps/chat/src/modules/chats/conversation/files/file-card.tsx create mode 100644 apps/chat/src/modules/chats/conversation/files/files-cards-list.tsx create mode 100644 apps/chat/src/modules/chats/conversation/files/index.ts create mode 100644 apps/chat/src/modules/chats/conversation/files/select-chat-file.tsx rename {apps/backend/src/modules/projects-embeddings => packages/commons/src/helpers}/mimetypes/index.ts (100%) rename {apps/backend/src/modules/projects-embeddings => packages/commons/src/helpers}/mimetypes/is-image-mimetype.ts (100%) rename {apps/backend/src/modules/projects-embeddings => packages/commons/src/helpers}/mimetypes/is-legacy-excel-mimetype.ts (100%) rename {apps/backend/src/modules/projects-embeddings => packages/commons/src/helpers}/mimetypes/is-legacy-word-mimetype.ts (100%) rename {apps/backend/src/modules/projects-embeddings => packages/commons/src/helpers}/mimetypes/is-pdf-mimetype.ts (100%) rename {apps/backend/src/modules/projects-embeddings => packages/commons/src/helpers}/mimetypes/is-xml-office-mimetype.ts (100%) create mode 100644 packages/commons/src/helpers/urls/is-pdf-file-url.ts create mode 100644 packages/commons/src/helpers/urls/is-word-file-url.ts diff --git a/apps/backend/src/modules/projects-embeddings/projects-embeddings.service.ts b/apps/backend/src/modules/projects-embeddings/projects-embeddings.service.ts index 1581285b..feefacb2 100644 --- a/apps/backend/src/modules/projects-embeddings/projects-embeddings.service.ts +++ b/apps/backend/src/modules/projects-embeddings/projects-embeddings.service.ts @@ -5,7 +5,14 @@ import isValidUTF8 from 'utf-8-validate'; import type { SdkJwtTokenT } from '@llm/sdk'; -import { isNil } from '@llm/commons'; +import { + isImageMimetype, + isLegacyExcelMimetype, + isLegacyWordMimetype, + isNil, + isPDFMimeType, + isXmlOfficeMimetype, +} from '@llm/commons'; import type { WithAuthFirewall } from '../auth'; import type { TableId, TableRowWithUuid } from '../database'; @@ -28,13 +35,6 @@ import { XlsAIEmbeddingGenerator, } from './generators'; import { createRelevantEmbeddingsPrompt, formatVector } from './helpers'; -import { - isImageMimetype, - isLegacyExcelMimetype, - isLegacyWordMimetype, - isPDFMimeType, - isXmlOfficeMimetype, -} from './mimetypes'; import { ProjectsEmbeddingsFirewall } from './projects-embeddings.firewall'; import { ProjectsEmbeddingsRepo } from './projects-embeddings.repo'; import { ProjectEmbeddingsInsertTableRow } from './projects-embeddings.tables'; diff --git a/apps/chat/src/modules/chats/conversation/chat-conversation-panel.tsx b/apps/chat/src/modules/chats/conversation/chat-conversation-panel.tsx index 50d8e3b8..ebb6397a 100644 --- a/apps/chat/src/modules/chats/conversation/chat-conversation-panel.tsx +++ b/apps/chat/src/modules/chats/conversation/chat-conversation-panel.tsx @@ -191,7 +191,11 @@ export const ChatConversationPanel = memo(({ ref, chat, initialMessages, classNa ref={messagesContainerRef} className={clsx( 'relative z-10 flex-1 [&::-webkit-scrollbar]:hidden p-4 [-ms-overflow-style:none] overflow-y-scroll [scrollbar-width:none]', - flickeringIndicator.visible ? 'opacity-100' : 'opacity-0', // Avoid scroll flickering on first render + + // Avoid scroll flickering on first render + flickeringIndicator.visible + ? 'opacity-100' + : 'opacity-0', )} onLoad={scrollConversation} > diff --git a/apps/chat/src/modules/chats/conversation/files/file-card.tsx b/apps/chat/src/modules/chats/conversation/files/file-card.tsx new file mode 100644 index 00000000..fb1971a8 --- /dev/null +++ b/apps/chat/src/modules/chats/conversation/files/file-card.tsx @@ -0,0 +1,122 @@ +import clsx from 'clsx'; +import { + FileAxis3DIcon, + FileIcon, + FileSpreadsheetIcon, + FileTextIcon, + ImageIcon, + XIcon, +} from 'lucide-react'; + +import { + isImageFileUrl, + isLegacyExcelMimetype, + isLegacyWordMimetype, + isPDFFileUrl, + isPDFMimeType, + isSpreadsheetFileUrl, + isWordFileUrl, +} from '@llm/commons'; +import { useI18n } from '~/i18n'; + +type Props = { + file: File; + onRemove: () => void; +}; + +export function FileCard({ file, onRemove }: Props) { + const { pack } = useI18n(); + const { type, bgColor, icon: IconComponent } = getFileTypeAndColor(file); + + const isImage = isImageFileUrl(file.name); + const imageUrl = isImage ? URL.createObjectURL(file) : null; + + return ( +
+ + + {isImage && imageUrl + ? ( + {file.name} + ) + : ( +
+
+ +
+ +
+ + {file.name} + + + + {type} + +
+
+ )} +
+ ); +} + +function getFileTypeAndColor(file: File) { + if (isSpreadsheetFileUrl(file.name) || isLegacyExcelMimetype(file.type)) { + return { + type: 'Excel', + bgColor: 'bg-[#217346]', + icon: FileSpreadsheetIcon, + }; + } + + if (isLegacyWordMimetype(file.type) || isWordFileUrl(file.name)) { + return { + type: 'Word', + bgColor: 'bg-[#2B579A]', + icon: FileTextIcon, + }; + } + + if (isPDFMimeType(file.type) || isPDFFileUrl(file.name)) { + return { + type: 'PDF', + bgColor: 'bg-[#D93F3F]', + icon: FileAxis3DIcon, + }; + } + + if (isImageFileUrl(file.name)) { + return { + type: 'Image', + bgColor: 'bg-purple-500', + icon: ImageIcon, + }; + } + + return { + type: 'TXT', + bgColor: 'bg-gray-400', + icon: FileIcon, + }; +} diff --git a/apps/chat/src/modules/chats/conversation/files/files-cards-list.tsx b/apps/chat/src/modules/chats/conversation/files/files-cards-list.tsx new file mode 100644 index 00000000..7a00de08 --- /dev/null +++ b/apps/chat/src/modules/chats/conversation/files/files-cards-list.tsx @@ -0,0 +1,59 @@ +import { controlled } from '@under-control/forms'; +import clsx from 'clsx'; +import { useMemo } from 'react'; +import { v4 } from 'uuid'; + +import { without } from '@llm/commons'; + +import { FileCard } from './file-card'; + +type Props = { + className?: string; +}; + +export const FilesCardsList = controlled(({ className, control: { value, setValue } }) => { + const mappedFiles = useMemo(() => { + const fileNames = new Set(); + const duplicateNames = new Set(); + + value.forEach((file) => { + if (fileNames.has(file.name)) { + duplicateNames.add(file.name); + } + + fileNames.add(file.name); + }); + + return value.map(file => ({ + id: duplicateNames.has(file.name) ? v4() : file.name, + file, + })); + }, [value]); + + if (!value?.length) { + return null; + } + + const handleRemove = (file: File) => { + setValue({ + value: without([file])(value), + }); + }; + + return ( +
+ {mappedFiles.map(({ file, id }) => ( + handleRemove(file)} + /> + ))} +
+ ); +}); diff --git a/apps/chat/src/modules/chats/conversation/files/index.ts b/apps/chat/src/modules/chats/conversation/files/index.ts new file mode 100644 index 00000000..d44668f2 --- /dev/null +++ b/apps/chat/src/modules/chats/conversation/files/index.ts @@ -0,0 +1,3 @@ +export * from './file-card'; +export * from './files-cards-list'; +export * from './select-chat-file'; diff --git a/apps/chat/src/modules/chats/conversation/files/select-chat-file.tsx b/apps/chat/src/modules/chats/conversation/files/select-chat-file.tsx new file mode 100644 index 00000000..4480184b --- /dev/null +++ b/apps/chat/src/modules/chats/conversation/files/select-chat-file.tsx @@ -0,0 +1,3 @@ +import { selectFile } from '@llm/commons'; + +export const selectChatFile = selectFile('.pdf,.csv,.doc,.docx,.xls,.xlsx,.png,.jpg,.jpeg,.bmp'); diff --git a/apps/chat/src/modules/chats/conversation/index.ts b/apps/chat/src/modules/chats/conversation/index.ts index 392cc47b..1545b869 100644 --- a/apps/chat/src/modules/chats/conversation/index.ts +++ b/apps/chat/src/modules/chats/conversation/index.ts @@ -1 +1,10 @@ +export * from './chat-attached-app'; +export * from './chat-background'; +export * from './chat-conversation-panel'; export * from './chat-conversation-with-sidebar'; +export * from './config-panel'; +export * from './content-parse'; +export * from './files'; +export * from './hooks'; +export * from './input-toolbar'; +export * from './messages'; diff --git a/apps/chat/src/modules/chats/start-chat/start-chat-form.tsx b/apps/chat/src/modules/chats/start-chat/start-chat-form.tsx index 24242e19..bd939d33 100644 --- a/apps/chat/src/modules/chats/start-chat/start-chat-form.tsx +++ b/apps/chat/src/modules/chats/start-chat/start-chat-form.tsx @@ -1,17 +1,19 @@ import type { KeyboardEventHandler } from 'react'; import clsx from 'clsx'; +import { pipe } from 'fp-ts/function'; import { PaperclipIcon, SendIcon } from 'lucide-react'; import type { SdkTableRowWithIdNameT } from '@llm/sdk'; -import { StrictBooleanV } from '@llm/commons'; +import { StrictBooleanV, tapTaskOption } from '@llm/commons'; import { useFocusAfterMount, useLocalStorageObject } from '@llm/commons-front'; import { Checkbox, FormSpinnerCTA } from '@llm/ui'; import { useI18n } from '~/i18n'; import { AIModelsSearchSelect } from '~/modules/ai-models'; import { ProjectsSearchSelect } from '~/modules/projects'; +import { FilesCardsList, selectChatFile } from '../conversation'; import { useStartChatForm } from './use-start-chat-form'; type Props = { @@ -42,6 +44,13 @@ export function StartChatForm({ forceProject, className }: Props) { } }; + const onAttachFile = pipe( + selectChatFile, + tapTaskOption((file) => { + bind.path('files').onChange([...value.files ?? [], file]); + }), + ); + return (
-