feat(telemetry): Telemetry MVP

Implements the first version of telemetry.

Created telemetry js package and added it to latitude's TS SDK.
Telemetry supports automatic telemetry data collection for the following providers:
OpenAI
Anthropic
Azure AI
Cohere
Google Vertex AI
Google AI Platform
AWS Bedrock
Vercel SDK (and all its providers)
Added backend to ingest traces and spans
Added frontend to visualize traces and spans
Added actions to automatically create prompts from spans
Added manual tracing with SDK's telemetry.span() method
Added automatic prompt log creation when a span is associated with a prompt

.tmuxinator.yml

@@ -1,6 +1,7 @@
 name: latitude-llm
 windows:
-  - web: cd apps/web
-  - apps: pnpm dev --filter='./apps/*' --filter='./packages/sdks/typescript' --filter='./packages/compiler'
+  - web: cd .
+  - apps: pnpm dev --filter='./apps/*'
+  - packages: pnpm dev --filter='./packages/*'
   - docker: docker compose up --menu=false
   - studio: cd packages/core && pnpm db:studio

apps/gateway/src/routes/api/v2/evaluations/handlers/getOrCreate.ts

@@ -0,0 +1,116 @@
+import { zValidator } from '@hono/zod-validator'
+import {
+  EvaluationMetadataType,
+  resultConfigurationSchema,
+} from '@latitude-data/core/browser'
+import { NotFoundError } from '@latitude-data/core/lib/errors'
+import {
+  CommitsRepository,
+  DocumentVersionsRepository,
+  EvaluationsRepository,
+} from '@latitude-data/core/repositories'
+import { connectEvaluations } from '@latitude-data/core/services/evaluations/connect'
+import { createAndConnect } from '@latitude-data/core/services/evaluations/createAndConnect'
+import { Factory } from 'hono/factory'
+import { z } from 'zod'
+
+const factory = new Factory()
+
+// Reuse the same schema structure from the createEvaluationAction
+const advancedEvaluationMetadataSchema = z.object({
+  type: z.literal(EvaluationMetadataType.LlmAsJudgeAdvanced),
+  prompt: z.string(),
+})
+const simpleEvaluationMetadataSchema = z.object({
+  type: z.literal(EvaluationMetadataType.LlmAsJudgeSimple),
+  providerApiKeyId: z.number().optional(),
+  model: z.string().optional(),
+  objective: z.string(),
+  additionalInstructions: z.string(),
+})
+const defaultEvaluationMetadataSchema = z.object({
+  type: z.literal(EvaluationMetadataType.Manual),
+})
+
+const getOrCreateSchema = z.object({
+  name: z.string(),
+  description: z.string(),
+  resultConfiguration: resultConfigurationSchema,
+  metadata: z.discriminatedUnion('type', [
+    advancedEvaluationMetadataSchema,
+    simpleEvaluationMetadataSchema,
+    defaultEvaluationMetadataSchema,
+  ]),
+  projectId: z.number().optional(),
+  promptPath: z.string().optional(),
+})
+
+export const getOrCreateHandler = factory.createHandlers(
+  zValidator('json', getOrCreateSchema),
+  async (c) => {
+    const workspace = c.get('workspace')
+    const user = c.get('user')
+    const input = c.req.valid('json')
+
+    // Try to find existing evaluation by name
+    const evaluationsScope = new EvaluationsRepository(workspace.id)
+    const existingEvaluation = await evaluationsScope
+      .findByName(input.name)
+      .then((r) => r.unwrap())
+
+    let documentUuid: string | undefined
+
+    if (input.promptPath && input.projectId) {
+      const documentsScope = new DocumentVersionsRepository(workspace.id)
+      const commitsScope = new CommitsRepository(workspace.id)
+      const headCommit = await commitsScope
+        .getHeadCommit(input.projectId)
+        .then((r) => r.unwrap())
+      if (!headCommit) {
+        throw new NotFoundError('No head commit found')
+      }
+
+      const document = await documentsScope
+        .getDocumentByPath({
+          commit: headCommit,
+          path: input.promptPath,
+        })
+        .then((r) => r.unwrap())
+      documentUuid = document.documentUuid
+    }
+
+    if (existingEvaluation) {
+      if (documentUuid) {
+        const connectResult = await connectEvaluations({
+          workspace,
+          documentUuid,
+          evaluationUuids: [existingEvaluation.uuid],
+          user,
+        })
+
+        if (connectResult.error) {
+          throw connectResult.error
+        }
+      }
+      return c.json(existingEvaluation)
+    }
+
+    const result = await createAndConnect({
+      workspace,
+      user,
+      name: input.name,
+      description: input.description,
+      metadataType: input.metadata.type,
+      metadata: input.metadata,
+      resultType: input.resultConfiguration.type,
+      resultConfiguration: input.resultConfiguration,
+      documentUuid,
+    })
+
+    if (result.error) {
+      throw result.error
+    }
+
+    return c.json(result.unwrap())
+  },
+)

apps/gateway/src/routes/api/v2/evaluations/index.ts

@@ -0,0 +1,7 @@
+import { Hono } from 'hono'
+
+import { getOrCreateHandler } from './handlers/getOrCreate'
+
+export const evaluationsRouter = new Hono()
+
+evaluationsRouter.post('/get-or-create', ...getOrCreateHandler)