feature: user generated logs

apps/gateway/package.json

@@ -27,7 +27,6 @@
     "@t3-oss/env-core": "^0.10.1",
     "drizzle-orm": "^0.33.0",
     "hono": "^4.5.3",
-    "jet-paths": "^1.0.6",
     "lodash-es": "^4.17.21",
     "zod": "^3.23.8",
     "rate-limiter-flexible": "^5.0.3"

apps/gateway/src/middlewares/errorHandler.ts

@@ -40,8 +40,8 @@ const errorHandlerMiddleware = (err: Error) => {
   if (err instanceof HTTPException) {
     return Response.json(
       {
-        name: ApiErrorCodes.HTTPExeption,
-        errorCode: ApiErrorCodes.HTTPExeption,
+        name: ApiErrorCodes.HTTPException,
+        errorCode: ApiErrorCodes.HTTPException,
         message: err.message,
         details: { cause: err.cause },
       },

apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/handlers/chat.ts

@@ -1,9 +1,8 @@
 import { zValidator } from '@hono/zod-validator'
-import { LogSources } from '@latitude-data/core/browser'
+import { LogSources, messagesSchema } from '@latitude-data/core/browser'
 import { streamToGenerator } from '@latitude-data/core/lib/streamToGenerator'
 import { addMessages } from '@latitude-data/core/services/documentLogs/index'
 import { captureException } from '@sentry/node'
-import { messageSchema } from '$/common/messageSchema'
 import { Factory } from 'hono/factory'
 import { streamSSE } from 'hono/streaming'
 import { z } from 'zod'
@@ -13,7 +12,7 @@ import { chainEventPresenter } from '../../../projects/[projectId]/versions/[ver
 const factory = new Factory()
 
 const schema = z.object({
-  messages: z.array(messageSchema),
+  messages: messagesSchema,
   __internal: z
     .object({
       source: z.nativeEnum(LogSources).optional(),

apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/index.ts

@@ -1,8 +1,7 @@
-import ROUTES from '$/common/routes'
 import { Hono } from 'hono'
 
 import { chatHandler } from './handlers/chat'
 
 export const chatsRouter = new Hono()
 
-chatsRouter.post(ROUTES.Api.V1.Conversations.Chat, ...chatHandler)
+chatsRouter.post('/:conversationUuid/chat', ...chatHandler)

apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/index.ts

@@ -1,12 +1,13 @@
-import ROUTES from '$/common/routes'
 import { Hono } from 'hono'
 
 import { getHandler } from './handlers/get'
 import { runHandler } from './handlers/run'
+import { logsRouterV1 } from './logs'
 
 const router = new Hono()
 
-router.get(ROUTES.Api.V1.Documents.Get, ...getHandler)
-router.post(ROUTES.Api.V1.Documents.Run, ...runHandler)
+router.get('/:documentPath{.+}', ...getHandler)
+router.post('/run', ...runHandler)
+router.route('/logs', logsRouterV1)
 
 export { router as documentsRouter }

apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts

@@ -0,0 +1,8 @@
+import { postHandler } from '$/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post'
+import { Hono } from 'hono'
+
+const router = new Hono()
+
+router.post('/', ...postHandler)
+
+export { router as logsRouterV1 }

apps/gateway/src/routes/api/v2/conversations/[conversationUuid]/index.ts

@@ -1,7 +1,6 @@
-import ROUTES from '$/common/routes'
 import { chatHandler } from '$/routes/api/v1/conversations/[conversationUuid]/handlers/chat'
 import { Hono } from 'hono'
 
 export const chatsRouter = new Hono()
 
-chatsRouter.post(ROUTES.Api.V2.Conversations.Chat, ...chatHandler)
+chatsRouter.post('/:conversationUuid/chat', ...chatHandler)

apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/index.ts

@@ -1,12 +1,13 @@
-import ROUTES from '$/common/routes'
 import { getHandler } from '$/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/handlers/get'
 import { Hono } from 'hono'
 
 import { runHandler } from './handlers/run'
+import { logsRouterV2 } from './logs'
 
 const router = new Hono()
 
-router.get(ROUTES.Api.V2.Documents.Get, ...getHandler)
-router.post(ROUTES.Api.V2.Documents.Run, ...runHandler)
+router.get('/:documentPath{.+}', ...getHandler)
+router.post('/run', ...runHandler)
+router.route('/logs', logsRouterV2)
 
 export { router as documentsRouter }

apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post.ts

@@ -0,0 +1,50 @@
+import { zValidator } from '@hono/zod-validator'
+import { LogSources, messagesSchema } from '@latitude-data/core/browser'
+import { generateUUIDIdentifier } from '@latitude-data/core/lib/generateUUID'
+import { createDocumentLog } from '@latitude-data/core/services/documentLogs/create'
+import { getData } from '$/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/handlers/_shared'
+import { Factory } from 'hono/factory'
+import { z } from 'zod'
+
+const factory = new Factory()
+
+const schema = z.object({
+  path: z.string(),
+  messages: messagesSchema,
+  response: z.string().optional(),
+})
+
+export const postHandler = factory.createHandlers(
+  zValidator('json', schema),
+  async (c) => {
+    const { projectId, versionUuid } = c.req.param()
+    const { path, messages, response } = c.req.valid('json')
+    const workspace = c.get('workspace')
+    const { document, commit } = await getData({
+      workspace,
+      projectId: Number(projectId!),
+      commitUuid: versionUuid!,
+      documentPath: path!,
+    }).then((r) => r.unwrap())
+
+    const documentLog = await createDocumentLog({
+      data: {
+        uuid: generateUUIDIdentifier(),
+        documentUuid: document.documentUuid,
+        resolvedContent: document.content,
+        source: LogSources.API,
+        parameters: {},
+        providerLog: {
+          messages,
+          responseText:
+            response ??
+            (messages[messages.length - 1].content?.text ||
+              messages[messages.length - 1].content),
+        },
+      },
+      commit,
+    }).then((r) => r.unwrap())
+
+    return c.json(documentLog)
+  },
+)

apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts

@@ -0,0 +1,9 @@
+import { Hono } from 'hono'
+
+import { postHandler } from './handlers/post'
+
+const router = new Hono()
+
+router.post('/', ...postHandler)
+
+export { router as logsRouterV2 }

apps/gateway/src/routes/app.ts

@@ -1,10 +1,8 @@
-import ROUTES from '$/common/routes'
 import authMiddleware from '$/middlewares/auth'
 import errorHandlerMiddleware from '$/middlewares/errorHandler'
 import rateLimitMiddleware from '$/middlewares/rateLimit'
 import { Hono } from 'hono'
 import { logger } from 'hono/logger'
-import jetPaths from 'jet-paths'
 
 import { chatsRouter as chatsRouterV1 } from './api/v1/conversations/[conversationUuid]'
 import { documentsRouter as documentsRouterV1 } from './api/v1/projects/[projectId]/versions/[versionUuid]/documents'
@@ -26,10 +24,19 @@ app.use(rateLimitMiddleware())
 app.use(authMiddleware())
 
 // Routers
-app.route(jetPaths(ROUTES).Api.V1.Documents.Base, documentsRouterV1)
-app.route(jetPaths(ROUTES).Api.V1.Conversations.Base, chatsRouterV1)
-app.route(jetPaths(ROUTES).Api.V2.Documents.Base, documentsRouterV2)
-app.route(jetPaths(ROUTES).Api.V2.Conversations.Base, chatsRouterV2)
+// v1
+app.route(
+  '/api/v1/projects/:projectId/versions/:versionUuid/documents',
+  documentsRouterV1,
+)
+app.route('/api/v1/conversations', chatsRouterV1)
+
+// v2
+app.route(
+  '/api/v2/projects/:projectId/versions/:versionUuid/documents',
+  documentsRouterV2,
+)
+app.route('/api/v2/conversations', chatsRouterV2)
 
 // Must be the last one!
 app.onError(errorHandlerMiddleware)

apps/web/src/actions/datasets/create.ts

@@ -1,27 +1,17 @@
 'use server'
 
+import {
+  DELIMITER_VALUES,
+  DELIMITERS_KEYS,
+  MAX_SIZE,
+  MAX_UPLOAD_SIZE_IN_MB,
+} from '@latitude-data/core/browser'
 import { DatasetsRepository } from '@latitude-data/core/repositories'
 import { createDataset } from '@latitude-data/core/services/datasets/create'
 import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
 
-const DELIMITERS_KEYS = [
-  'comma',
-  'semicolon',
-  'tab',
-  'space',
-  'custom',
-] as const
-const DELIMITER_VALUES = {
-  comma: ',',
-  semicolon: ';',
-  tab: '\t',
-  space: ' ',
-}
-
-const MAX_SIZE = 15
-const MAX_UPLOAD_SIZE_IN_MB = MAX_SIZE * 1024 * 1024
 export const createDatasetAction = authProcedure
   .createServerAction()
   .input(

apps/web/src/actions/documentLogs/upload.ts

@@ -0,0 +1,50 @@
+'use server'
+
+import {
+  DELIMITERS_KEYS,
+  MAX_SIZE,
+  MAX_UPLOAD_SIZE_IN_MB,
+} from '@latitude-data/core/browser'
+import { CommitsRepository } from '@latitude-data/core/repositories'
+import { bulkUploadDocumentLogs } from '@latitude-data/core/services/documentLogs/bulkUpload'
+import { z } from 'zod'
+
+import { withDocument } from '../procedures'
+
+export const uploadDocumentLogsAction = withDocument
+  .createServerAction()
+  .input(
+    z.object({
+      csvDelimiter: z.enum(DELIMITERS_KEYS, {
+        message: 'Choose a valid delimiter option',
+      }),
+      logsFile: z
+        .instanceof(File)
+        .refine((file) => {
+          return !file || file.size <= MAX_UPLOAD_SIZE_IN_MB
+        }, `Your file must be less than ${MAX_SIZE}MB in size`)
+        .refine(
+          (file) => file.type === 'text/csv',
+          'Your file must be a CSV file',
+        ),
+    }),
+  )
+  .handler(async ({ ctx, input }) => {
+    const commitsScope = new CommitsRepository(ctx.workspace.id)
+    const commit = await commitsScope
+      .getCommitByUuid({
+        projectId: ctx.project.id,
+        uuid: ctx.currentCommitUuid,
+      })
+      .then((r) => r.unwrap())
+
+    await bulkUploadDocumentLogs({
+      workspace: ctx.workspace,
+      document: ctx.document,
+      commit,
+      csvDelimiter: input.csvDelimiter,
+      logsFile: input.logsFile,
+    })
+
+    return { success: true }
+  })