feat(core): user logs

- Introduced a new job `uploadDocumentLogsJob` to handle bulk uploads of document logs from CSV files.
- Added `createDocumentLogJob` to create individual document logs from parsed CSV data.
- Updated `messageSchema` and `messagesSchema` to validate message structures.
- Implemented `bulkUploadDocumentLogs` service to parse CSV files and enqueue jobs for processing.
- Modified `ProviderLogsRepository` to include additional joins for better data retrieval.
- Enhanced `createProviderLog` to handle optional fields and avoid errors when certain data is missing.
- Updated various components to handle nullable fields and provide default values where necessary.
- Added new constants for delimiter values and maximum upload size.
- Updated SDK to include a new `log` method for logging messages and responses.

These changes enable users to upload large sets of document logs in bulk, improving efficiency and user experience. The new job definitions and services ensure that the logs are processed asynchronously, reducing the load on the main application.

apps/gateway/package.json

@@ -26,7 +26,6 @@
     "@t3-oss/env-core": "^0.10.1",
     "drizzle-orm": "^0.33.0",
     "hono": "^4.5.3",
-    "jet-paths": "^1.0.6",
     "lodash-es": "^4.17.21",
     "zod": "^3.23.8",
     "rate-limiter-flexible": "^5.0.3"

apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/handlers/chat.ts

@@ -1,9 +1,8 @@
 import { zValidator } from '@hono/zod-validator'
-import { LogSources } from '@latitude-data/core/browser'
+import { LogSources, messagesSchema } from '@latitude-data/core/browser'
 import { streamToGenerator } from '@latitude-data/core/lib/streamToGenerator'
 import { addMessages } from '@latitude-data/core/services/documentLogs/index'
 import { captureException } from '@sentry/node'
-import { messageSchema } from '$/common/messageSchema'
 import { Factory } from 'hono/factory'
 import { streamSSE } from 'hono/streaming'
 import { z } from 'zod'
@@ -13,7 +12,7 @@ import { chainEventPresenter } from '../../../projects/[projectId]/versions/[ver
 const factory = new Factory()
 
 const schema = z.object({
-  messages: z.array(messageSchema),
+  messages: messagesSchema,
   __internal: z
     .object({
       source: z.nativeEnum(LogSources).optional(),

apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/index.ts

@@ -1,8 +1,7 @@
-import ROUTES from '$/common/routes'
 import { Hono } from 'hono'
 
 import { chatHandler } from './handlers/chat'
 
 export const chatsRouter = new Hono()
 
-chatsRouter.post(ROUTES.Api.V1.Conversations.Chat, ...chatHandler)
+chatsRouter.post('/:conversationUuid/chat', ...chatHandler)

apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/index.ts

@@ -1,12 +1,13 @@
-import ROUTES from '$/common/routes'
 import { Hono } from 'hono'
 
 import { getHandler } from './handlers/get'
 import { runHandler } from './handlers/run'
+import { logsRouterV1 } from './logs'
 
 const router = new Hono()
 
-router.get(ROUTES.Api.V1.Documents.Get, ...getHandler)
-router.post(ROUTES.Api.V1.Documents.Run, ...runHandler)
+router.get('/:documentPath{.+}', ...getHandler)
+router.post('/run', ...runHandler)
+router.route('/logs', logsRouterV1)
 
 export { router as documentsRouter }

apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts

@@ -0,0 +1,8 @@
+import { postHandler } from '$/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post'
+import { Hono } from 'hono'
+
+const router = new Hono()
+
+router.post('/', ...postHandler)
+
+export { router as logsRouterV1 }

apps/gateway/src/routes/api/v2/conversations/[conversationUuid]/index.ts

@@ -1,7 +1,6 @@
-import ROUTES from '$/common/routes'
 import { chatHandler } from '$/routes/api/v1/conversations/[conversationUuid]/handlers/chat'
 import { Hono } from 'hono'
 
 export const chatsRouter = new Hono()
 
-chatsRouter.post(ROUTES.Api.V2.Conversations.Chat, ...chatHandler)
+chatsRouter.post('/:conversationUuid/chat', ...chatHandler)

apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/index.ts

@@ -1,12 +1,13 @@
-import ROUTES from '$/common/routes'
 import { getHandler } from '$/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/handlers/get'
 import { Hono } from 'hono'
 
 import { runHandler } from './handlers/run'
+import { logsRouterV2 } from './logs'
 
 const router = new Hono()
 
-router.get(ROUTES.Api.V2.Documents.Get, ...getHandler)
-router.post(ROUTES.Api.V2.Documents.Run, ...runHandler)
+router.get('/:documentPath{.+}', ...getHandler)
+router.post('/run', ...runHandler)
+router.route('/logs', logsRouterV2)
 
 export { router as documentsRouter }

apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post.ts

@@ -0,0 +1,50 @@
+import { zValidator } from '@hono/zod-validator'
+import { LogSources, messagesSchema } from '@latitude-data/core/browser'
+import { generateUUIDIdentifier } from '@latitude-data/core/lib/generateUUID'
+import { createDocumentLog } from '@latitude-data/core/services/documentLogs/create'
+import { getData } from '$/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/handlers/_shared'
+import { Factory } from 'hono/factory'
+import { z } from 'zod'
+
+const factory = new Factory()
+
+const schema = z.object({
+  path: z.string(),
+  messages: messagesSchema,
+  response: z.string().optional(),
+})
+
+export const postHandler = factory.createHandlers(
+  zValidator('json', schema),
+  async (c) => {
+    const { projectId, versionUuid } = c.req.param()
+    const { path, messages, response } = c.req.valid('json')
+    const workspace = c.get('workspace')
+    const { document, commit } = await getData({
+      workspace,
+      projectId: Number(projectId!),
+      commitUuid: versionUuid!,
+      documentPath: path!,
+    }).then((r) => r.unwrap())
+
+    const documentLog = await createDocumentLog({
+      data: {
+        uuid: generateUUIDIdentifier(),
+        documentUuid: document.documentUuid,
+        resolvedContent: document.content,
+        source: LogSources.API,
+        parameters: {},
+        providerLog: {
+          messages,
+          responseText:
+            response ??
+            (messages[messages.length - 1].content?.text ||
+              messages[messages.length - 1].content),
+        },
+      },
+      commit,
+    }).then((r) => r.unwrap())
+
+    return c.json(documentLog)
+  },
+)

apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts

@@ -0,0 +1,9 @@
+import { Hono } from 'hono'
+
+import { postHandler } from './handlers/post'
+
+const router = new Hono()
+
+router.post('/', ...postHandler)
+
+export { router as logsRouterV2 }

apps/gateway/src/routes/app.ts

@@ -1,10 +1,8 @@
-import ROUTES from '$/common/routes'
 import authMiddleware from '$/middlewares/auth'
 import errorHandlerMiddleware from '$/middlewares/errorHandler'
 import rateLimitMiddleware from '$/middlewares/rateLimit'
 import { Hono } from 'hono'
 import { logger } from 'hono/logger'
-import jetPaths from 'jet-paths'
 
 import { chatsRouter as chatsRouterV1 } from './api/v1/conversations/[conversationUuid]'
 import { documentsRouter as documentsRouterV1 } from './api/v1/projects/[projectId]/versions/[versionUuid]/documents'
@@ -26,10 +24,19 @@ app.use(rateLimitMiddleware())
 app.use(authMiddleware())
 
 // Routers
-app.route(jetPaths(ROUTES).Api.V1.Documents.Base, documentsRouterV1)
-app.route(jetPaths(ROUTES).Api.V1.Conversations.Base, chatsRouterV1)
-app.route(jetPaths(ROUTES).Api.V2.Documents.Base, documentsRouterV2)
-app.route(jetPaths(ROUTES).Api.V2.Conversations.Base, chatsRouterV2)
+// v1
+app.route(
+  '/api/v1/projects/:projectId/versions/:versionUuid/documents',
+  documentsRouterV1,
+)
+app.route('/api/v1/conversations', chatsRouterV1)
+
+// v2
+app.route(
+  '/api/v2/projects/:projectId/versions/:versionUuid/documents',
+  documentsRouterV2,
+)
+app.route('/api/v2/conversations', chatsRouterV2)
 
 // Must be the last one!
 app.onError(errorHandlerMiddleware)

apps/web/src/actions/datasets/create.ts

@@ -1,27 +1,17 @@
 'use server'
 
+import {
+  DELIMITER_VALUES,
+  DELIMITERS_KEYS,
+  MAX_SIZE,
+  MAX_UPLOAD_SIZE_IN_MB,
+} from '@latitude-data/core/browser'
 import { DatasetsRepository } from '@latitude-data/core/repositories'
 import { createDataset } from '@latitude-data/core/services/datasets/create'
 import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
 
-const DELIMITERS_KEYS = [
-  'comma',
-  'semicolon',
-  'tab',
-  'space',
-  'custom',
-] as const
-const DELIMITER_VALUES = {
-  comma: ',',
-  semicolon: ';',
-  tab: '\t',
-  space: ' ',
-}
-
-const MAX_SIZE = 15
-const MAX_UPLOAD_SIZE_IN_MB = MAX_SIZE * 1024 * 1024
 export const createDatasetAction = authProcedure
   .createServerAction()
   .input(

apps/web/src/actions/documentLogs/upload.ts

@@ -0,0 +1,50 @@
+'use server'
+
+import {
+  DELIMITERS_KEYS,
+  MAX_SIZE,
+  MAX_UPLOAD_SIZE_IN_MB,
+} from '@latitude-data/core/browser'
+import { CommitsRepository } from '@latitude-data/core/repositories'
+import { bulkUploadDocumentLogs } from '@latitude-data/core/services/documentLogs/bulkUpload'
+import { z } from 'zod'
+
+import { withDocument } from '../procedures'
+
+export const uploadDocumentLogsAction = withDocument
+  .createServerAction()
+  .input(
+    z.object({
+      csvDelimiter: z.enum(DELIMITERS_KEYS, {
+        message: 'Choose a valid delimiter option',
+      }),
+      logsFile: z
+        .instanceof(File)
+        .refine((file) => {
+          return !file || file.size <= MAX_UPLOAD_SIZE_IN_MB
+        }, `Your file must be less than ${MAX_SIZE}MB in size`)
+        .refine(
+          (file) => file.type === 'text/csv',
+          'Your file must be a CSV file',
+        ),
+    }),
+  )
+  .handler(async ({ ctx, input }) => {
+    const commitsScope = new CommitsRepository(ctx.workspace.id)
+    const commit = await commitsScope
+      .getCommitByUuid({
+        projectId: ctx.project.id,
+        uuid: ctx.currentCommitUuid,
+      })
+      .then((r) => r.unwrap())
+
+    await bulkUploadDocumentLogs({
+      workspace: ctx.workspace,
+      document: ctx.document,
+      commit,
+      csvDelimiter: input.csvDelimiter,
+      logsFile: input.logsFile,
+    })
+
+    return { success: true }
+  })