diff --git a/apps/gateway/package.json b/apps/gateway/package.json index 51b62132e..2d54db2dd 100644 --- a/apps/gateway/package.json +++ b/apps/gateway/package.json @@ -16,6 +16,7 @@ "@hono/zod-validator": "^0.2.2", "@latitude-data/core": "workspace:^", "@latitude-data/env": "workspace:^", + "@latitude-data/jobs": "workspace:^", "@t3-oss/env-core": "^0.10.1", "hono": "^4.5.3", "zod": "^3.23.8" diff --git a/apps/gateway/src/common/env.ts b/apps/gateway/src/common/env.ts index 4fa552ff0..6998f4af9 100644 --- a/apps/gateway/src/common/env.ts +++ b/apps/gateway/src/common/env.ts @@ -3,16 +3,11 @@ import '@latitude-data/env' import { createEnv } from '@t3-oss/env-core' import { z } from 'zod' -let env +let localEnv = {} if (process.env.NODE_ENV === 'development') { - env = await import('./env/development').then((r) => r.default) + localEnv = await import('./env/development').then((r) => r.default) } else if (process.env.NODE_ENV === 'test') { - env = await import('./env/test').then((r) => r.default) -} else { - env = process.env as { - GATEWAY_PORT: string - GATEWAY_HOST: string - } + localEnv = await import('./env/test').then((r) => r.default) } export default createEnv({ @@ -21,6 +16,12 @@ export default createEnv({ server: { GATEWAY_PORT: z.string().optional().default('8787'), GATEWAY_HOST: z.string().optional().default('localhost'), + REDIS_HOST: z.string(), + REDIS_PORT: z.string(), + REDIS_PASSWORD: z.string().optional(), + }, + runtimeEnv: { + ...process.env, + ...localEnv, }, - runtimeEnv: env, }) diff --git a/apps/gateway/src/jobs/index.ts b/apps/gateway/src/jobs/index.ts new file mode 100644 index 000000000..201c53d64 --- /dev/null +++ b/apps/gateway/src/jobs/index.ts @@ -0,0 +1,10 @@ +import { setupJobs } from '@latitude-data/jobs' +import env from '$/common/env' + +export const { queues } = setupJobs({ + connectionParams: { + host: env.REDIS_HOST, + port: Number(env.REDIS_PORT), + password: env.REDIS_PASSWORD, + }, +}) diff --git a/apps/gateway/src/middlewares/auth.ts b/apps/gateway/src/middlewares/auth.ts index 128baa51e..a5f85c751 100644 --- a/apps/gateway/src/middlewares/auth.ts +++ b/apps/gateway/src/middlewares/auth.ts @@ -23,6 +23,7 @@ const authMiddleware = () => if (workspaceResult.error) return false c.set('workspace', workspaceResult.value) + c.set('apiKey', apiKeyResult.value) return true }, diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/_shared.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/_shared.ts index aa83b45d2..54a90aee8 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/_shared.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/_shared.ts @@ -5,14 +5,6 @@ import { } from '@latitude-data/core' import type { Workspace } from '@latitude-data/core/browser' -const toDocumentPath = (path: string) => { - if (path.startsWith('/')) { - return path - } - - return `/${path}` -} - export const getData = async ({ workspace, projectId, @@ -35,10 +27,7 @@ export const getData = async ({ .getCommitByUuid({ project, uuid: commitUuid }) .then((r) => r.unwrap()) const document = await docsScope - .getDocumentByPath({ - commit, - path: toDocumentPath(documentPath), - }) + .getDocumentByPath({ commit, path: documentPath }) .then((r) => r.unwrap()) return { project, commit, document } diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts index 7d04afeec..877cc728b 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts @@ -7,7 +7,11 @@ import { } from '@latitude-data/core' import app from '$/index' import { eq } from 'drizzle-orm' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$/jobs', () => ({ + queues: { jobs: { enqueueUpdateApiKeyProviderJob: vi.fn() } }, +})) describe('GET documents', () => { describe('unauthorized', () => { @@ -26,7 +30,7 @@ describe('GET documents', () => { const apikey = await database.query.apiKeys.findFirst({ where: eq(apiKeys.workspaceId, workspace.id), }) - const path = '/path/to/document' + const path = 'path/to/document' const { commit } = await factories.createDraft({ project, user, @@ -42,7 +46,7 @@ describe('GET documents', () => { .getDocumentByPath({ commit, path }) .then((r) => r.unwrap()) - const route = `/api/v1/projects/${project!.id}/commits/${commit!.uuid}/documents/${document.documentVersion.path.slice(1)}` + const route = `/api/v1/projects/${project!.id}/commits/${commit!.uuid}/documents/${document.documentVersion.path}` const res = await app.request(route, { headers: { Authorization: `Bearer ${apikey!.token}`, diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts index d226d4e26..2a1569eac 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts @@ -12,6 +12,7 @@ import { describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ runDocumentAtCommit: vi.fn(), + queues: { jobs: { enqueueUpdateApiKeyProviderJob: vi.fn() } }, })) vi.mock('@latitude-data/core', async (importOriginal) => { @@ -23,6 +24,10 @@ vi.mock('@latitude-data/core', async (importOriginal) => { } }) +vi.mock('$/jobs', () => ({ + queues: mocks.queues, +})) + describe('POST /run', () => { describe('unauthorized', () => { it('fails', async () => { @@ -112,6 +117,7 @@ describe('POST /run', () => { }, }) + expect(mocks.queues) expect(res.status).toBe(200) expect(res.body).toBeInstanceOf(ReadableStream) diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts index 504c7a08b..f4674c425 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts @@ -1,5 +1,10 @@ import { zValidator } from '@hono/zod-validator' -import { runDocumentAtCommit } from '@latitude-data/core' +import { + LogSources, + runDocumentAtCommit, + streamToGenerator, +} from '@latitude-data/core' +import { queues } from '$/jobs' import { Factory } from 'hono/factory' import { SSEStreamingApi, streamSSE } from 'hono/streaming' import { z } from 'zod' @@ -21,6 +26,7 @@ export const runHandler = factory.createHandlers( const { documentPath, parameters } = c.req.valid('json') const workspace = c.get('workspace') + const apiKey = c.get('apiKey') const { document, commit } = await getData({ workspace, @@ -33,6 +39,13 @@ export const runHandler = factory.createHandlers( documentUuid: document.documentUuid, commit, parameters, + logHandler: (log) => { + queues.defaultQueue.jobs.enqueueCreateProviderLogJob({ + ...log, + source: LogSources.API, + apiKeyId: apiKey.id, + }) + }, }).then((r) => r.unwrap()) await pipeToStream(stream, result.stream) @@ -45,12 +58,7 @@ async function pipeToStream( readableStream: ReadableStream, ) { let id = 0 - const reader = readableStream.getReader() - - while (true) { - const { done, value } = await reader.read() - if (done) break - + for await (const value of streamToGenerator(readableStream)) { stream.write( JSON.stringify({ ...value, diff --git a/apps/web/package.json b/apps/web/package.json index 9b6bcaf10..23565b1ab 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -27,6 +27,7 @@ "ai": "^3.2.42", "bcrypt": "^5.1.1", "bullmq": "^5.8.5", + "date-fns": "^3.6.0", "ioredis": "^5.4.1", "lodash-es": "^4.17.21", "lucia": "^3.2.0", diff --git a/apps/web/src/actions/documents/streamTextAction.ts b/apps/web/src/actions/documents/streamTextAction.ts index b6c7ae5b4..3480ed6e5 100644 --- a/apps/web/src/actions/documents/streamTextAction.ts +++ b/apps/web/src/actions/documents/streamTextAction.ts @@ -7,7 +7,8 @@ import { streamToGenerator, validateConfig, } from '@latitude-data/core' -import { PROVIDER_EVENT } from '@latitude-data/core/browser' +import { LogSources, PROVIDER_EVENT } from '@latitude-data/core/browser' +import { queues } from '$/jobs' import { getCurrentUser } from '$/services/auth/getCurrentUser' import { createStreamableValue, StreamableValue } from 'ai/rsc' @@ -26,7 +27,7 @@ export async function streamTextAction({ messages, }: StreamTextActionProps): StreamTextActionResponse { const { workspace } = await getCurrentUser() - const { provider, model, ...rest } = validateConfig(config) + const { provider, ...rest } = validateConfig(config) const providerApiKeysScope = new ProviderApiKeysRepository(workspace.id) const apiKey = await providerApiKeysScope .findByName(provider) @@ -35,13 +36,21 @@ export async function streamTextAction({ ;(async () => { try { - const result = await ai({ - apiKey: apiKey.token, - provider: apiKey.provider, - model, - messages, - config: rest, - }) + const result = await ai( + { + provider: apiKey, + messages, + config: rest, + }, + { + logHandler: (log) => { + queues.defaultQueue.jobs.enqueueCreateProviderLogJob({ + ...log, + source: LogSources.Playground, + }) + }, + }, + ) for await (const value of streamToGenerator(result.fullStream)) { stream.update({ diff --git a/apps/web/src/app/(private)/settings/_components/ProviderApiKeys/index.tsx b/apps/web/src/app/(private)/settings/_components/ProviderApiKeys/index.tsx index 846cf3bbe..ddf300b4c 100644 --- a/apps/web/src/app/(private)/settings/_components/ProviderApiKeys/index.tsx +++ b/apps/web/src/app/(private)/settings/_components/ProviderApiKeys/index.tsx @@ -16,9 +16,22 @@ import { } from '@latitude-data/web-ui' import useProviderApiKeys from '$/stores/providerApiKeys' import useUsers from '$/stores/users' +import { format, formatDistanceToNow, formatRelative } from 'date-fns' import NewApiKey from './New' +const HOURS = 1000 * 60 * 60 +const DAYS = HOURS * 24 +function relativeTime(date: Date | null) { + if (date == null) return 'never' + + const now = new Date() + const diff = now.getTime() - date.getTime() + if (diff < 1 * HOURS) return formatDistanceToNow(date, { addSuffix: true }) + if (diff < 7 * DAYS) return formatRelative(date, new Date()) + return format(date, 'PPpp') +} + export default function ProviderApiKeys() { const { data: providerApiKeys, destroy } = useProviderApiKeys() const [open, setOpen] = useState(false) @@ -97,7 +110,7 @@ const ProviderApiKeysTable = ({ - {apiKey.lastUsedAt?.toISOString() || 'never'} + {relativeTime(apiKey.lastUsedAt)} diff --git a/apps/web/src/jobs/index.ts b/apps/web/src/jobs/index.ts index 73dfb1edf..b5d3af27b 100644 --- a/apps/web/src/jobs/index.ts +++ b/apps/web/src/jobs/index.ts @@ -1,7 +1,7 @@ import { setupJobs } from '@latitude-data/jobs' import env from '$/env' -export default setupJobs({ +export const { queues } = setupJobs({ connectionParams: { host: env.REDIS_HOST, port: Number(env.REDIS_PORT), diff --git a/packages/core/drizzle/0026_latitude_rocks.sql b/packages/core/drizzle/0026_latitude_rocks.sql new file mode 100644 index 000000000..877c72566 --- /dev/null +++ b/packages/core/drizzle/0026_latitude_rocks.sql @@ -0,0 +1,63 @@ +DO $$ BEGIN + CREATE TYPE "latitude"."log_source" AS ENUM('playground', 'api'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "latitude"."document_logs" ( + "id" bigserial PRIMARY KEY NOT NULL, + "uuid" uuid NOT NULL, + "document_uuid" uuid NOT NULL, + "commit_id" bigint NOT NULL, + "resolved_content" text NOT NULL, + "parameters" json NOT NULL, + "custom_identifier" text, + "duration" bigint NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "document_logs_uuid_unique" UNIQUE("uuid") +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "latitude"."provider_logs" ( + "id" bigserial PRIMARY KEY NOT NULL, + "uuid" uuid NOT NULL, + "provider_id" bigint NOT NULL, + "model" varchar, + "config" json NOT NULL, + "messages" json NOT NULL, + "response_text" text, + "tool_calls" json, + "tokens" bigint NOT NULL, + "duration" bigint NOT NULL, + "document_log_id" bigint, + "source" "latitude"."log_source" NOT NULL, + "apiKeyId" bigint, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "provider_logs_uuid_unique" UNIQUE("uuid") +); +--> statement-breakpoint +ALTER TABLE "latitude"."api_keys" ADD COLUMN "last_used_at" timestamp;--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "latitude"."document_logs" ADD CONSTRAINT "document_logs_commit_id_commits_id_fk" FOREIGN KEY ("commit_id") REFERENCES "latitude"."commits"("id") ON DELETE restrict ON UPDATE cascade; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "latitude"."provider_logs" ADD CONSTRAINT "provider_logs_provider_id_provider_api_keys_id_fk" FOREIGN KEY ("provider_id") REFERENCES "latitude"."provider_api_keys"("id") ON DELETE restrict ON UPDATE cascade; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "latitude"."provider_logs" ADD CONSTRAINT "provider_logs_document_log_id_document_logs_id_fk" FOREIGN KEY ("document_log_id") REFERENCES "latitude"."document_logs"("id") ON DELETE restrict ON UPDATE cascade; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "latitude"."provider_logs" ADD CONSTRAINT "provider_logs_apiKeyId_api_keys_id_fk" FOREIGN KEY ("apiKeyId") REFERENCES "latitude"."api_keys"("id") ON DELETE restrict ON UPDATE cascade; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; diff --git a/packages/core/drizzle/meta/0026_snapshot.json b/packages/core/drizzle/meta/0026_snapshot.json new file mode 100644 index 000000000..c6c29a147 --- /dev/null +++ b/packages/core/drizzle/meta/0026_snapshot.json @@ -0,0 +1,1101 @@ +{ + "id": "6467cdb8-e5f0-4a74-9aef-4e59c3977766", + "prevId": "18a8ad06-3115-4ac8-8a33-15659b9a447b", + "version": "7", + "dialect": "postgresql", + "tables": { + "latitude.users": { + "name": "users", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_password": { + "name": "encrypted_password", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + }, + "latitude.sessions": { + "name": "sessions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.workspaces": { + "name": "workspaces", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_creator_id_users_id_fk": { + "name": "workspaces_creator_id_users_id_fk", + "tableFrom": "workspaces", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.memberships": { + "name": "memberships", + "schema": "latitude", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_workspace_id_workspaces_id_fk": { + "name": "memberships_workspace_id_workspaces_id_fk", + "tableFrom": "memberships", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_workspace_id_user_id_pk": { + "name": "memberships_workspace_id_user_id_pk", + "columns": [ + "workspace_id", + "user_id" + ] + } + }, + "uniqueConstraints": {} + }, + "latitude.api_keys": { + "name": "api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_id_idx": { + "name": "workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_keys_workspace_id_workspaces_id_fk": { + "name": "api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_token_unique": { + "name": "api_keys_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.projects": { + "name": "projects", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_idx": { + "name": "workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "projects_workspace_id_workspaces_id_fk": { + "name": "projects_workspace_id_workspaces_id_fk", + "tableFrom": "projects", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.commits": { + "name": "commits", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "project_commit_order_idx": { + "name": "project_commit_order_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "commits_project_id_projects_id_fk": { + "name": "commits_project_id_projects_id_fk", + "tableFrom": "commits", + "tableTo": "projects", + "schemaTo": "latitude", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "commits_user_id_users_id_fk": { + "name": "commits_user_id_users_id_fk", + "tableFrom": "commits", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "commits_uuid_unique": { + "name": "commits_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + }, + "unique_commit_version": { + "name": "unique_commit_version", + "nullsNotDistinct": false, + "columns": [ + "version", + "project_id" + ] + } + } + }, + "latitude.document_versions": { + "name": "document_versions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "document_versions_commit_id_commits_id_fk": { + "name": "document_versions_commit_id_commits_id_fk", + "tableFrom": "document_versions", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "unique_document_uuid_commit_id": { + "name": "unique_document_uuid_commit_id", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "commit_id" + ] + }, + "unique_path_commit_id_deleted_at": { + "name": "unique_path_commit_id_deleted_at", + "nullsNotDistinct": false, + "columns": [ + "path", + "commit_id", + "deleted_at" + ] + } + } + }, + "latitude.provider_api_keys": { + "name": "provider_api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "provider_apikeys_workspace_id_idx": { + "name": "provider_apikeys_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_name_idx": { + "name": "provider_apikeys_name_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_user_id_idx": { + "name": "provider_apikeys_user_id_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_api_keys_author_id_users_id_fk": { + "name": "provider_api_keys_author_id_users_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_api_keys_workspace_id_workspaces_id_fk": { + "name": "provider_api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_apikeys_token_provider_unique": { + "name": "provider_apikeys_token_provider_unique", + "nullsNotDistinct": false, + "columns": [ + "token", + "provider", + "workspace_id" + ] + } + } + }, + "latitude.document_logs": { + "name": "document_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parameters": { + "name": "parameters", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "custom_identifier": { + "name": "custom_identifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "document_logs_commit_id_commits_id_fk": { + "name": "document_logs_commit_id_commits_id_fk", + "tableFrom": "document_logs", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "document_logs_uuid_unique": { + "name": "document_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.provider_logs": { + "name": "provider_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "messages": { + "name": "messages", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "response_text": { + "name": "response_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "document_log_id": { + "name": "document_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "apiKeyId": { + "name": "apiKeyId", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_logs_provider_id_provider_api_keys_id_fk": { + "name": "provider_logs_provider_id_provider_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "provider_api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "provider_logs_document_log_id_document_logs_id_fk": { + "name": "provider_logs_document_log_id_document_logs_id_fk", + "tableFrom": "provider_logs", + "tableTo": "document_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "document_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "provider_logs_apiKeyId_api_keys_id_fk": { + "name": "provider_logs_apiKeyId_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "apiKeyId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_logs_uuid_unique": { + "name": "provider_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + } + }, + "enums": { + "latitude.provider": { + "name": "provider", + "schema": "latitude", + "values": [ + "openai", + "anthropic", + "groq", + "mistral", + "azure" + ] + }, + "latitude.log_source": { + "name": "log_source", + "schema": "latitude", + "values": [ + "playground", + "api" + ] + } + }, + "schemas": { + "latitude": "latitude" + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/core/drizzle/meta/_journal.json b/packages/core/drizzle/meta/_journal.json index 0c918ddd1..abf7d5890 100644 --- a/packages/core/drizzle/meta/_journal.json +++ b/packages/core/drizzle/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1722928026330, "tag": "0025_illegal_stark_industries", "breakpoints": true + }, + { + "idx": 26, + "version": "7", + "when": 1723110315363, + "tag": "0026_latitude_rocks", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/core/package.json b/packages/core/package.json index b5db7e897..e325a3949 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -32,9 +32,10 @@ "@t3-oss/env-core": "^0.10.1", "ai": "^3.2.42", "bcrypt": "^5.1.1", - "drizzle-orm": "0.31.4", "drizzle-kit": "^0.22.8", + "drizzle-orm": "0.31.4", "lodash-es": "^4.17.21", + "uuid": "^10.0.0", "zod": "^3.23.8" }, "devDependencies": { @@ -44,6 +45,7 @@ "@types/bcrypt": "^5.0.2", "@types/lodash-es": "^4.17.12", "@types/node": "^20.14.10", + "@types/uuid": "^10.0.0", "eslint-plugin-drizzle": "^0.2.3", "pg-transactional-tests": "^1.0.9", "supertest": "^7.0.0", diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index a060e0695..2aef15b23 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -32,6 +32,11 @@ export enum Providers { Azure = 'azure', } +export enum LogSources { + Playground = 'playground', + API = 'api', +} + export const PROVIDER_EVENT = 'provider-event' export const LATITUDE_EVENT = 'latitude-event' diff --git a/packages/core/src/schema/index.ts b/packages/core/src/schema/index.ts index d94aa4550..6c1f63a01 100644 --- a/packages/core/src/schema/index.ts +++ b/packages/core/src/schema/index.ts @@ -13,3 +13,7 @@ export * from './models/commits' export * from './models/documentVersions' export * from './models/providerApiKeys' + +// Log tables +export * from './models/documentLogs' +export * from './models/providerLogs' diff --git a/packages/core/src/schema/models/apiKeys.ts b/packages/core/src/schema/models/apiKeys.ts index 7d7a8b564..a7cec4743 100644 --- a/packages/core/src/schema/models/apiKeys.ts +++ b/packages/core/src/schema/models/apiKeys.ts @@ -24,6 +24,7 @@ export const apiKeys = latitudeSchema.table( .notNull() .references(() => workspaces.id), name: varchar('name', { length: 256 }), + lastUsedAt: timestamp('last_used_at'), deletedAt: timestamp('deleted_at'), ...timestamps(), }, diff --git a/packages/core/src/schema/models/documentLogs.ts b/packages/core/src/schema/models/documentLogs.ts new file mode 100644 index 000000000..0a53f89ee --- /dev/null +++ b/packages/core/src/schema/models/documentLogs.ts @@ -0,0 +1,37 @@ +import { relations } from 'drizzle-orm' +import { bigint, bigserial, json, text, uuid } from 'drizzle-orm/pg-core' + +import { latitudeSchema } from '../db-schema' +import { timestamps } from '../schemaHelpers' +import { commits } from './commits' +import { providerLogs } from './providerLogs' + +export const documentLogs = latitudeSchema.table('document_logs', { + id: bigserial('id', { mode: 'number' }).notNull().primaryKey(), + uuid: uuid('uuid').notNull().unique(), + documentUuid: uuid('document_uuid').notNull(), // document_uuid cannot be a reference to document_versions, because it is not a unique field + commitId: bigint('commit_id', { mode: 'number' }) + .notNull() + .references(() => commits.id, { + onDelete: 'restrict', + onUpdate: 'cascade', + }), + resolvedContent: text('resolved_content').notNull(), + parameters: json('parameters').notNull(), + customIdentifier: text('custom_identifier'), + duration: bigint('duration', { mode: 'number' }).notNull(), + ...timestamps(), +}) + +export const documentLogsRelations = relations( + documentLogs, + ({ one, many }) => ({ + commit: one(commits, { + fields: [documentLogs.commitId], + references: [commits.id], + }), + providerLogs: many(providerLogs, { + relationName: 'providerLogDocumentLog', + }), + }), +) diff --git a/packages/core/src/schema/models/providerLogs.ts b/packages/core/src/schema/models/providerLogs.ts new file mode 100644 index 000000000..52a6fb3b6 --- /dev/null +++ b/packages/core/src/schema/models/providerLogs.ts @@ -0,0 +1,71 @@ +import { LogSources } from '$core/constants' +import { relations } from 'drizzle-orm' +import { + bigint, + bigserial, + json, + text, + uuid, + varchar, +} from 'drizzle-orm/pg-core' + +import { apiKeys } from '..' +import { latitudeSchema } from '../db-schema' +import { timestamps } from '../schemaHelpers' +import { documentLogs } from './documentLogs' +import { providerApiKeys } from './providerApiKeys' + +export const logSourcesEnum = latitudeSchema.enum('log_source', [ + LogSources.Playground, + LogSources.API, +]) + +export const providerLogs = latitudeSchema.table('provider_logs', { + id: bigserial('id', { mode: 'number' }).notNull().primaryKey(), + uuid: uuid('uuid').notNull().unique(), + providerId: bigint('provider_id', { mode: 'number' }) + .notNull() + .references(() => providerApiKeys.id, { + onDelete: 'restrict', + onUpdate: 'cascade', + }), + model: varchar('model'), + config: json('config').notNull(), + messages: json('messages').notNull(), + responseText: text('response_text'), + toolCalls: json('tool_calls'), + tokens: bigint('tokens', { mode: 'number' }).notNull(), + duration: bigint('duration', { mode: 'number' }).notNull(), + documentLogId: bigint('document_log_id', { mode: 'number' }).references( + () => documentLogs.id, + { + onDelete: 'restrict', + onUpdate: 'cascade', + }, + ), + source: logSourcesEnum('source').notNull(), + apiKeyId: bigint('apiKeyId', { mode: 'number' }).references( + () => apiKeys.id, + { + onDelete: 'restrict', + onUpdate: 'cascade', + }, + ), + ...timestamps(), +}) + +export const providerLogsRelations = relations(providerLogs, ({ one }) => ({ + provider: one(providerApiKeys, { + fields: [providerLogs.providerId], + references: [providerApiKeys.id], + }), + documentLog: one(documentLogs, { + relationName: 'providerLogDocumentLog', + fields: [providerLogs.documentLogId], + references: [documentLogs.id], + }), + apiKey: one(apiKeys, { + fields: [providerLogs.apiKeyId], + references: [apiKeys.id], + }), +})) diff --git a/packages/core/src/schema/types.ts b/packages/core/src/schema/types.ts index e8d206f7a..c7382d5cd 100644 --- a/packages/core/src/schema/types.ts +++ b/packages/core/src/schema/types.ts @@ -9,6 +9,9 @@ import { users } from '$core/schema/models/users' import { workspaces } from '$core/schema/models/workspaces' import { type InferSelectModel } from 'drizzle-orm' +import { documentLogs } from './models/documentLogs' +import { providerLogs } from './models/providerLogs' + // Model types are out of schema files to be able to share with NextJS webpack bundler // otherwise, it will throw an error. export type Workspace = InferSelectModel @@ -23,3 +26,5 @@ export type ApiKey = InferSelectModel export type Commit = InferSelectModel export type DocumentVersion = InferSelectModel export type Project = InferSelectModel +export type ProviderLog = InferSelectModel +export type DocumentLog = InferSelectModel diff --git a/packages/core/src/services/ai/index.ts b/packages/core/src/services/ai/index.ts index 6b7aa11c4..d0f46a416 100644 --- a/packages/core/src/services/ai/index.ts +++ b/packages/core/src/services/ai/index.ts @@ -4,7 +4,7 @@ import { createMistral } from '@ai-sdk/mistral' import { createOpenAI } from '@ai-sdk/openai' import { OpenAICompletionModelId } from '@ai-sdk/openai/internal' import { Message } from '@latitude-data/compiler' -import { Providers } from '$core/browser' +import { ProviderApiKey, Providers } from '$core/browser' import { CallWarning, CompletionTokenUsage, @@ -12,9 +12,12 @@ import { FinishReason, streamText, } from 'ai' +import { v4 as uuidv4 } from 'uuid' import { z } from 'zod' -type FinishCallbackEvent = { +import { CreateProviderLogProps } from '../providerLogs' + +export type FinishCallbackEvent = { finishReason: FinishReason usage: CompletionTokenUsage text: string @@ -39,7 +42,7 @@ export type Config = { model: string azure?: { resourceName: string } } & Record -export type PartialConfig = Omit +export type PartialConfig = Omit const GROQ_API_URL = 'https://api.groq.com/openai/v1' @@ -82,35 +85,56 @@ function createProvider({ } } +export type AILog = Omit + export async function ai( { + provider: apiProvider, prompt, messages, - apiKey, - model, - provider, config, }: { - prompt?: string + provider: ProviderApiKey + config: PartialConfig messages: Message[] - apiKey: string - model: OpenAICompletionModelId - config?: PartialConfig - provider: Providers + prompt?: string }, { + logHandler, onFinish, }: { + logHandler: (log: AILog) => void onFinish?: FinishCallback - } = {}, + }, ) { + const startTime = Date.now() + const { provider, token: apiKey, id: providerId } = apiProvider + const model = config.model as OpenAICompletionModelId const m = createProvider({ provider, apiKey, config })(model) return await streamText({ model: m, prompt, messages: messages as CoreMessage[], - onFinish, + onFinish: (event) => { + logHandler({ + logUuid: uuidv4(), + providerId, + model, + config, + messages, + responseText: event.text, + toolCalls: event.toolCalls?.map((t) => ({ + id: t.toolCallId, + name: t.toolName, + arguments: t.args, + })), + tokens: event.usage.totalTokens, + duration: Date.now() - startTime, + }) + + onFinish?.(event) + }, }) } diff --git a/packages/core/src/services/apiKeys/index.ts b/packages/core/src/services/apiKeys/index.ts index f756a8bb3..c5634425e 100644 --- a/packages/core/src/services/apiKeys/index.ts +++ b/packages/core/src/services/apiKeys/index.ts @@ -1 +1,2 @@ export * from './create' +export * from './touch' diff --git a/packages/core/src/services/apiKeys/touch.ts b/packages/core/src/services/apiKeys/touch.ts new file mode 100644 index 000000000..0d27e0c18 --- /dev/null +++ b/packages/core/src/services/apiKeys/touch.ts @@ -0,0 +1,21 @@ +import { database } from '$core/client' +import { NotFoundError, Result, Transaction } from '$core/lib' +import { apiKeys } from '$core/schema' +import { eq } from 'drizzle-orm' + +export function touchApiKey(id: number, db = database) { + return Transaction.call(async (tx) => { + const result = await tx + .update(apiKeys) + .set({ + lastUsedAt: new Date(), + }) + .where(eq(apiKeys.id, id)) + .returning() + + if (!result.length) { + return Result.error(new NotFoundError('ApiKey not found')) + } + return Result.ok(result[0]!) + }, db) +} diff --git a/packages/core/src/services/commits/runDocumentAtCommit.ts b/packages/core/src/services/commits/runDocumentAtCommit.ts index 3d11bba70..748d7e247 100644 --- a/packages/core/src/services/commits/runDocumentAtCommit.ts +++ b/packages/core/src/services/commits/runDocumentAtCommit.ts @@ -12,17 +12,19 @@ import { NotFoundError, Result } from '$core/lib' import { streamToGenerator } from '$core/lib/streamToGenerator' import { ProviderApiKeysRepository } from '$core/repositories' -import { ai, validateConfig } from '../ai' +import { ai, AILog, validateConfig } from '../ai' import { createChainAtCommit } from './createChainAtCommit' export async function runDocumentAtCommit({ documentUuid, commit, parameters, + logHandler, }: { documentUuid: string commit: Commit parameters: Record + logHandler: (log: AILog) => void }) { const workspace = await findWorkspaceFromCommit(commit) if (!workspace) throw Result.error(new NotFoundError('Workspace not found')) @@ -44,7 +46,7 @@ export async function runDocumentAtCommit({ await new Promise((resolve) => { stream = new ReadableStream({ start(controller) { - response = iterate({ chain, scope, controller }) + response = iterate({ chain, scope, controller, logHandler }) resolve() }, @@ -64,6 +66,7 @@ async function iterate({ controller, previousCount = 0, previousResponse, + logHandler, }: { chain: Chain scope: ProviderApiKeysRepository @@ -74,6 +77,7 @@ async function iterate({ text: string usage: Record } + logHandler: (log: AILog) => void }) { try { const { conversation, completed, config, apiKey, sentCount } = @@ -97,12 +101,14 @@ async function iterate({ event: LATITUDE_EVENT, }) - const result = await ai({ - messages: conversation.messages, - apiKey: apiKey.token, - provider: apiKey.provider, - model: config.model, - }) + const result = await ai( + { + messages: conversation.messages, + config: config, + provider: apiKey, + }, + { logHandler }, + ) for await (const value of streamToGenerator(result.fullStream)) { controller.enqueue({ @@ -143,6 +149,7 @@ async function iterate({ previousApiKey: apiKey, previousCount: sentCount, previousResponse: response, + logHandler, }) } } catch (error) { diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 2c6113a07..ca4131d08 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -7,3 +7,4 @@ export * from './workspaces' export * from './providerApiKeys' export * from './apiKeys' export * from './ai' +export * from './providerLogs' diff --git a/packages/core/src/services/providerApiKeys/index.ts b/packages/core/src/services/providerApiKeys/index.ts index 42f1ddcbf..0300de0cb 100644 --- a/packages/core/src/services/providerApiKeys/index.ts +++ b/packages/core/src/services/providerApiKeys/index.ts @@ -1,2 +1,3 @@ export * from './create' export * from './destroy' +export * from './touch' diff --git a/packages/core/src/services/providerApiKeys/touch.ts b/packages/core/src/services/providerApiKeys/touch.ts new file mode 100644 index 000000000..fed401154 --- /dev/null +++ b/packages/core/src/services/providerApiKeys/touch.ts @@ -0,0 +1,21 @@ +import { database } from '$core/client' +import { NotFoundError, Result, Transaction } from '$core/lib' +import { providerApiKeys } from '$core/schema' +import { eq } from 'drizzle-orm' + +export function touchProviderApiKey(id: number, db = database) { + return Transaction.call(async (tx) => { + const result = await tx + .update(providerApiKeys) + .set({ + lastUsedAt: new Date(), + }) + .where(eq(providerApiKeys.id, id)) + .returning() + + if (!result.length) { + return Result.error(new NotFoundError('ProviderApiKey not found')) + } + return Result.ok(result[0]!) + }, db) +} diff --git a/packages/core/src/services/providerLogs/index.ts b/packages/core/src/services/providerLogs/index.ts new file mode 100644 index 000000000..91580251d --- /dev/null +++ b/packages/core/src/services/providerLogs/index.ts @@ -0,0 +1,67 @@ +import { Message, ToolCall } from '@latitude-data/compiler' +import { + database, + providerLogs, + Result, + touchApiKey, + Transaction, +} from '@latitude-data/core' +import { LogSources, ProviderLog } from '$core/browser' + +import { touchProviderApiKey } from '../providerApiKeys/touch' + +export type CreateProviderLogProps = { + logUuid: string + providerId: number + model: string + config: Record + messages: Message[] + responseText: string + toolCalls?: ToolCall[] + tokens: number + duration: number + source: LogSources + apiKeyId?: number +} + +export async function createProviderLog( + { + logUuid, + providerId, + model, + config, + messages, + responseText, + toolCalls, + tokens, + duration, + source, + apiKeyId, + }: CreateProviderLogProps, + db = database, +) { + return Transaction.call(async (trx) => { + const inserts = await trx + .insert(providerLogs) + .values({ + uuid: logUuid, + providerId, + model, + config, + messages, + responseText, + toolCalls, + tokens, + duration, + source, + apiKeyId, + }) + .returning() + + const log = inserts[0]! + await touchProviderApiKey(providerId, trx) + if (apiKeyId) await touchApiKey(apiKeyId, trx) + + return Result.ok(log) + }, db) +} diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index 133abb9eb..2398e27ce 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -12,6 +12,5 @@ if (env !== 'production') { ELASTIC_PASSWORD: 'secret', REDIS_PORT: '6379', REDIS_HOST: 'localhost', - REDIS_PASSWORD: 'secret', }) } diff --git a/packages/jobs/package.json b/packages/jobs/package.json index c095b6884..4d344d7d7 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "description": "Latitude jobs from Latitude llm", "main": "./src/index.ts", + "type": "module", "scripts": { "dev": "tsx watch ./src/server.ts", "build:server": "rollup -c", @@ -13,6 +14,7 @@ }, "dependencies": { "@latitude-data/env": "workspace:*", + "@latitude-data/core": "workspace:*", "bullmq": "^5.8.3", "ioredis": "^5.4.1", "zod": "^3.23.8" diff --git a/packages/jobs/src/constants.ts b/packages/jobs/src/constants.ts index b6cf384c8..712fca9c6 100644 --- a/packages/jobs/src/constants.ts +++ b/packages/jobs/src/constants.ts @@ -1,7 +1,7 @@ export enum Queues { - exampleQueue = 'exampleQueue', + defaultQueue = 'defaultQueue', } export enum Jobs { - exampleJob = 'exampleJob', + createProviderLogJob = 'createProviderLogJob', } diff --git a/packages/jobs/src/job-definitions/index.ts b/packages/jobs/src/job-definitions/index.ts index e5ebbe840..175c5d020 100644 --- a/packages/jobs/src/job-definitions/index.ts +++ b/packages/jobs/src/job-definitions/index.ts @@ -1,9 +1,9 @@ import { Jobs, Queues } from '$jobs/constants' -export type ExampleJobData = { patata: string } +import { CreateProviderLogJobData } from './providerLogs/createJob' -type JobData = J extends Jobs.exampleJob - ? ExampleJobData +type JobData = J extends Jobs.createProviderLogJob + ? CreateProviderLogJobData : never type JobSpec = { @@ -12,7 +12,7 @@ type JobSpec = { } export type JobDefinition = { - [Queues.exampleQueue]: { - [Jobs.exampleJob]: JobSpec + [Queues.defaultQueue]: { + [Jobs.createProviderLogJob]: JobSpec } } diff --git a/packages/jobs/src/job-definitions/providerLogs/createJob.ts b/packages/jobs/src/job-definitions/providerLogs/createJob.ts new file mode 100644 index 000000000..008461e42 --- /dev/null +++ b/packages/jobs/src/job-definitions/providerLogs/createJob.ts @@ -0,0 +1,10 @@ +import { createProviderLog, CreateProviderLogProps } from '@latitude-data/core' +import { Job } from 'bullmq' + +export type CreateProviderLogJobData = CreateProviderLogProps + +export const createProviderLogJob = async ( + job: Job, +) => { + await createProviderLog(job.data).then((r) => r.unwrap()) +} diff --git a/packages/jobs/src/queues/index.ts b/packages/jobs/src/queues/index.ts index 15b461adf..0640b541e 100644 --- a/packages/jobs/src/queues/index.ts +++ b/packages/jobs/src/queues/index.ts @@ -44,10 +44,10 @@ function setupQueue({ export function setupQueues({ connection }: { connection: Redis }) { return { - [Queues.exampleQueue]: setupQueue({ + [Queues.defaultQueue]: setupQueue({ connection, - name: Queues.exampleQueue, - jobs: [Jobs.exampleJob], + name: Queues.defaultQueue, + jobs: [Jobs.createProviderLogJob], }), } } diff --git a/packages/jobs/src/workers/index.ts b/packages/jobs/src/workers/index.ts index 735f0a1d3..8d47a67f3 100644 --- a/packages/jobs/src/workers/index.ts +++ b/packages/jobs/src/workers/index.ts @@ -1,7 +1,7 @@ import { Worker } from 'bullmq' import { Redis } from 'ioredis' -import { default as exampleWorker } from './worker-definitions/exampleWorker' +import { defaultWorker } from './worker-definitions/defaultWorker' const WORKER_OPTS = { concurrency: 5, @@ -10,7 +10,7 @@ const WORKER_OPTS = { removeOnFail: { count: 0 }, } -const WORKERS = [exampleWorker] +const WORKERS = [defaultWorker] export default function startWorkers({ connection }: { connection: Redis }) { return WORKERS.map((w) => { diff --git a/packages/jobs/src/workers/worker-definitions/defaultWorker.ts b/packages/jobs/src/workers/worker-definitions/defaultWorker.ts new file mode 100644 index 000000000..5954c330e --- /dev/null +++ b/packages/jobs/src/workers/worker-definitions/defaultWorker.ts @@ -0,0 +1,20 @@ +import { Jobs, Queues } from '$jobs/constants' +import { + createProviderLogJob, + CreateProviderLogJobData, +} from '$jobs/job-definitions/providerLogs/createJob' +import { Job, Processor } from 'bullmq' + +const processor: Processor = async (job) => { + switch (job.name) { + case Jobs.createProviderLogJob: + return await createProviderLogJob(job as Job) + default: + // do nothing + } +} + +export const defaultWorker = { + processor, + queueName: Queues.defaultQueue, +} diff --git a/packages/jobs/src/workers/worker-definitions/exampleWorker.ts b/packages/jobs/src/workers/worker-definitions/exampleWorker.ts deleted file mode 100644 index 636bd454e..000000000 --- a/packages/jobs/src/workers/worker-definitions/exampleWorker.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Queues } from '$jobs/constants' -import { ExampleJobData } from '$jobs/job-definitions' -import { Job, Processor } from 'bullmq' - -type ExampleResult = { job: Job; patata: string } -const processor: Processor = async (job) => { - console.log('JOB received', job.id, job.data) - const { patata } = job.data - - await new Promise((resolve) => setTimeout(resolve, 5000)) - - console.log('Running run', job.id, patata) - return { job, patata } -} - -export default { - processor, - queueName: Queues.exampleQueue, -} diff --git a/packages/jobs/tsconfig.json b/packages/jobs/tsconfig.json index da1a5edd7..99487e106 100644 --- a/packages/jobs/tsconfig.json +++ b/packages/jobs/tsconfig.json @@ -1,11 +1,17 @@ { "extends": "@latitude-data/typescript-config/base.json", "compilerOptions": { + "moduleResolution": "Bundler", "baseUrl": ".", "rootDir": "./src", "outDir": "./dist", "paths": { - "$jobs/*": ["./src/*"] + "$jobs/*": ["./src/*"], + "@latitude-data/core": ["../core/src/*"], + "@latitude-data/compiler": ["../compiler/src/*"], + "$core/*": ["../core/src/*"], + "$compiler/*": ["../compiler/src/*"], + "acorn": ["node_modules/@latitude-data/typescript-config/types/acorn"] } }, "include": ["src/**/*"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee64a37ba..f74a24cc4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -63,6 +63,9 @@ importers: '@latitude-data/env': specifier: workspace:^ version: link:../../packages/env + '@latitude-data/jobs': + specifier: workspace:^ + version: link:../../packages/jobs '@t3-oss/env-core': specifier: ^0.10.1 version: 0.10.1(typescript@5.5.4)(zod@3.23.8) @@ -133,6 +136,9 @@ importers: bullmq: specifier: ^5.8.5 version: 5.8.7 + date-fns: + specifier: ^3.6.0 + version: 3.6.0 ioredis: specifier: ^5.4.1 version: 5.4.1 @@ -315,6 +321,9 @@ importers: lodash-es: specifier: ^4.17.21 version: 4.17.21 + uuid: + specifier: ^10.0.0 + version: 10.0.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -337,6 +346,9 @@ importers: '@types/node': specifier: ^20.14.10 version: 20.14.10 + '@types/uuid': + specifier: ^10.0.0 + version: 10.0.0 eslint-plugin-drizzle: specifier: ^0.2.3 version: 0.2.3(eslint@9.6.0) @@ -371,6 +383,9 @@ importers: packages/jobs: dependencies: + '@latitude-data/core': + specifier: workspace:* + version: link:../core '@latitude-data/env': specifier: workspace:* version: link:../env @@ -3838,6 +3853,10 @@ packages: '@types/send': 0.17.4 dev: true + /@types/uuid@10.0.0: + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + dev: true + /@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0)(eslint@9.6.0)(typescript@5.5.3): resolution: {integrity: sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA==} engines: {node: ^16.0.0 || >=18.0.0} @@ -5140,6 +5159,10 @@ packages: is-data-view: 1.0.1 dev: true + /date-fns@3.6.0: + resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} + dev: false + /debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -9647,6 +9670,11 @@ packages: /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + /uuid@10.0.0: + resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + hasBin: true + dev: false + /uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true