diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/get.test.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/get.test.ts index b8bd89e2c..e594289b5 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/get.test.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/get.test.ts @@ -40,6 +40,8 @@ describe('GET documents', () => { user, }) const document = await createDocumentVersion({ + workspace, + user, commit, path, content: helpers.createPrompt({ provider: providers[0]! }), diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/run.test.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/run.test.ts index 543231626..42520018c 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/run.test.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/versions/:versionUuid/documents/handlers/run.test.ts @@ -95,6 +95,8 @@ describe('POST /run', () => { user, }) const document = await createDocumentVersion({ + workspace, + user, commit: cmt, path, content: helpers.createPrompt({ provider: providers[0]! }), diff --git a/apps/web/src/actions/commits/deleteDraftCommit.test.ts b/apps/web/src/actions/commits/deleteDraftCommit.test.ts index 63ece949e..61f057b8c 100644 --- a/apps/web/src/actions/commits/deleteDraftCommit.test.ts +++ b/apps/web/src/actions/commits/deleteDraftCommit.test.ts @@ -117,11 +117,15 @@ describe('getUsersAction', () => { user, }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'patata/doc1', content: factories.helpers.createPrompt({ provider }), }) await factories.createDocumentVersion({ + workspace, + user, commit: anotherDraf, path: 'patata/doc2', content: factories.helpers.createPrompt({ provider }), diff --git a/apps/web/src/actions/commits/publishDraftCommitAction.ts b/apps/web/src/actions/commits/publishDraftCommitAction.ts index 60cedc534..ba22b9ce4 100644 --- a/apps/web/src/actions/commits/publishDraftCommitAction.ts +++ b/apps/web/src/actions/commits/publishDraftCommitAction.ts @@ -1,5 +1,6 @@ 'use server' +import { publisher } from '@latitude-data/core/events/publisher' import { CommitsRepository } from '@latitude-data/core/repositories' import { mergeCommit } from '@latitude-data/core/services/commits/merge' import { z } from 'zod' @@ -15,5 +16,15 @@ export const publishDraftCommitAction = withProject .getCommitById(input.id) .then((r) => r.unwrap()) const merged = await mergeCommit(commit).then((r) => r.unwrap()) + + publisher.publishLater({ + type: 'commitPublished', + data: { + commit: merged, + userEmail: ctx.user.email, + workspaceId: ctx.workspace.id, + }, + }) + return merged }) diff --git a/apps/web/src/actions/connectedEvaluations/update.test.ts b/apps/web/src/actions/connectedEvaluations/update.test.ts index 670913d43..24877908a 100644 --- a/apps/web/src/actions/connectedEvaluations/update.test.ts +++ b/apps/web/src/actions/connectedEvaluations/update.test.ts @@ -39,7 +39,10 @@ describe('updateConnectedEvaluationAction', () => { user = setup.user // Create a connected evaluation using a factory - const evaluation = await factories.createLlmAsJudgeEvaluation({ workspace }) + const evaluation = await factories.createLlmAsJudgeEvaluation({ + workspace, + user, + }) const { commit } = await factories.createDraft({ project, user }) const { documentLog } = await factories.createDocumentLog({ document: setup.documents[0]!, @@ -47,6 +50,7 @@ describe('updateConnectedEvaluationAction', () => { }) connectedEvaluation = await factories.createConnectedEvaluation({ + user, workspace, evaluationUuid: evaluation.uuid, documentUuid: documentLog.documentUuid, diff --git a/apps/web/src/actions/datasets/create.ts b/apps/web/src/actions/datasets/create.ts index ec575a0d4..c4034acf6 100644 --- a/apps/web/src/actions/datasets/create.ts +++ b/apps/web/src/actions/datasets/create.ts @@ -74,6 +74,7 @@ export const createDatasetAction = authProcedure input.csvDelimiter === 'custom' ? input.csvCustomDelimiter : DELIMITER_VALUES[input.csvDelimiter] + return createDataset({ workspace: ctx.workspace, author: ctx.user, diff --git a/apps/web/src/actions/documents/create.ts b/apps/web/src/actions/documents/create.ts index 139e2ffbc..3bbe6cf15 100644 --- a/apps/web/src/actions/documents/create.ts +++ b/apps/web/src/actions/documents/create.ts @@ -22,6 +22,8 @@ export const createDocumentVersionAction = withProject .then((r) => r.unwrap()) const result = await createNewDocument({ + workspace: ctx.workspace, + user: ctx.user, commit, path: input.path, }) diff --git a/apps/web/src/actions/evaluations/connect.test.ts b/apps/web/src/actions/evaluations/connect.test.ts index d0554c93d..42a201b3a 100644 --- a/apps/web/src/actions/evaluations/connect.test.ts +++ b/apps/web/src/actions/evaluations/connect.test.ts @@ -73,6 +73,7 @@ describe('connectEvaluationsAction', () => { it('connects evaluations and templates to a document', async () => { const evaluation = await factories.createLlmAsJudgeEvaluation({ workspace, + user, name: 'Test Evaluation', prompt: factories.helpers.createPrompt({ provider }), }) diff --git a/apps/web/src/actions/evaluations/connect.ts b/apps/web/src/actions/evaluations/connect.ts index 61da116b0..09ea41280 100644 --- a/apps/web/src/actions/evaluations/connect.ts +++ b/apps/web/src/actions/evaluations/connect.ts @@ -20,6 +20,7 @@ export const connectEvaluationsAction = withProject documentUuid: input.documentUuid, evaluationUuids: input.evaluationUuids, templateIds: input.templateIds, + user: ctx.user, }).then((r) => r.unwrap()) return connectedEvaluations diff --git a/apps/web/src/actions/evaluations/create.ts b/apps/web/src/actions/evaluations/create.ts index 6412ce0e7..7fc8e2c58 100644 --- a/apps/web/src/actions/evaluations/create.ts +++ b/apps/web/src/actions/evaluations/create.ts @@ -41,6 +41,7 @@ export const createEvaluationAction = authProcedure metadata: input.metadata, configuration: input.configuration, type: input.type, + user: ctx.user, }) return result.unwrap() diff --git a/apps/web/src/actions/evaluations/destroy.test.ts b/apps/web/src/actions/evaluations/destroy.test.ts index bff7323fa..0ea191573 100644 --- a/apps/web/src/actions/evaluations/destroy.test.ts +++ b/apps/web/src/actions/evaluations/destroy.test.ts @@ -34,6 +34,7 @@ describe('destroyEvaluationAction', () => { }) const evaluation = await factories.createLlmAsJudgeEvaluation({ workspace, + user: userData, prompt: factories.helpers.createPrompt({ provider }), }) evaluationId = evaluation.id @@ -68,6 +69,7 @@ describe('destroyEvaluationAction', () => { }) evaluation = await factories.createLlmAsJudgeEvaluation({ workspace, + user, prompt: factories.helpers.createPrompt({ provider }), }) diff --git a/apps/web/src/actions/evaluations/runBatch.test.ts b/apps/web/src/actions/evaluations/runBatch.test.ts index 384bf5c8a..4cce82896 100644 --- a/apps/web/src/actions/evaluations/runBatch.test.ts +++ b/apps/web/src/actions/evaluations/runBatch.test.ts @@ -91,6 +91,7 @@ describe('runBatchAction', () => { .then((result) => result.dataset) evaluation = await factories.createLlmAsJudgeEvaluation({ + user, workspace, name: 'Test Evaluation', }) @@ -179,6 +180,7 @@ describe('runBatchAction', () => { it('enqueues multiple evaluation jobs for multiple evaluationIds', async () => { const evaluation2 = await factories.createLlmAsJudgeEvaluation({ + user, workspace, name: 'Test Evaluation 2', }) diff --git a/apps/web/src/actions/evaluations/runBatch.ts b/apps/web/src/actions/evaluations/runBatch.ts index ed3696099..a00aa7b77 100644 --- a/apps/web/src/actions/evaluations/runBatch.ts +++ b/apps/web/src/actions/evaluations/runBatch.ts @@ -98,7 +98,10 @@ export const runBatchEvaluationAction = withDataset const queues = await setupJobs() evaluations.forEach((evaluation) => { const batchId = `evaluation:${evaluation.id}:${nanoid(5)}` + queues.defaultQueue.jobs.enqueueRunBatchEvaluationJob({ + workspace: ctx.workspace, + user: ctx.user, evaluation, dataset: ctx.dataset, document: ctx.document, diff --git a/apps/web/src/actions/providerApiKeys/create.ts b/apps/web/src/actions/providerApiKeys/create.ts index 181994b88..ab94cf4df 100644 --- a/apps/web/src/actions/providerApiKeys/create.ts +++ b/apps/web/src/actions/providerApiKeys/create.ts @@ -22,7 +22,7 @@ export const createProviderApiKeyAction = authProcedure provider: input.provider, token: input.token, name: input.name, - authorId: ctx.user.id, + author: ctx.user, }) .then((r) => r.unwrap()) .then(providerApiKeyPresenter) diff --git a/apps/web/src/actions/providerApiKeys/fetch.test.ts b/apps/web/src/actions/providerApiKeys/fetch.test.ts index d1630538f..0303f15e6 100644 --- a/apps/web/src/actions/providerApiKeys/fetch.test.ts +++ b/apps/web/src/actions/providerApiKeys/fetch.test.ts @@ -46,8 +46,8 @@ describe('getProviderApiKeyAction', async () => { // openai example apikey token token: 'sk-1234567890abcdef1234567890abcdef', workspace: session!.workspace, + author: session!.userData, name: 'foo', - authorId: session!.userData.id, }) const [data, error] = await getProviderApiKeyAction() diff --git a/apps/web/src/actions/user/setupAction.ts b/apps/web/src/actions/user/setupAction.ts index 00104e254..4c3824f05 100644 --- a/apps/web/src/actions/user/setupAction.ts +++ b/apps/web/src/actions/user/setupAction.ts @@ -35,6 +35,7 @@ export const setupAction = errorHandlingProcedure .handler(async ({ input }) => { const result = await setupService(input) const { user } = result.unwrap() + await createMagicLinkToken({ user: user }).then((r) => r.unwrap()) redirect(ROUTES.auth.magicLinkSent(user.email)) diff --git a/apps/web/src/app/api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations/route.test.ts b/apps/web/src/app/api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations/route.test.ts index 593e7af41..1c8314126 100644 --- a/apps/web/src/app/api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations/route.test.ts +++ b/apps/web/src/app/api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations/route.test.ts @@ -47,10 +47,12 @@ describe('GET /api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations mockEvaluations = [ await factories.createConnectedEvaluation({ workspace, + user: setup.user, documentUuid: documents[0]!.documentUuid, }), await factories.createConnectedEvaluation({ workspace, + user: setup.user, documentUuid: documents[1]!.documentUuid, }), ] diff --git a/apps/web/src/services/user/setupService.ts b/apps/web/src/services/user/setupService.ts index c1ca09b18..64626d3ff 100644 --- a/apps/web/src/services/user/setupService.ts +++ b/apps/web/src/services/user/setupService.ts @@ -47,7 +47,7 @@ export default function setupService({ provider: Providers.OpenAI, name: env.DEFAULT_PROVIDER_ID, token: env.DEFAULT_PROVIDER_API_KEY, - authorId: user.id, + author: user, }, tx, ) diff --git a/package.json b/package.json index 4d6006fcf..c351fefe8 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "nodemailer-html-to-text": "^3.2.0", "nodemailer-mailgun-transport": "^2.1.5", "pg": "^8.12.0", + "posthog-node": "^4.2.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-resizable": "^3.0.5", diff --git a/packages/core/drizzle/0060_omniscient_gunslinger.sql b/packages/core/drizzle/0060_omniscient_gunslinger.sql new file mode 100644 index 000000000..a57e5d035 --- /dev/null +++ b/packages/core/drizzle/0060_omniscient_gunslinger.sql @@ -0,0 +1,24 @@ +/* + Unfortunately in current drizzle-kit version we can't automatically get name for primary key. + We are working on making it available! + + Meanwhile you can: + 1. Check pk name in your database, by running + SELECT constraint_name FROM information_schema.table_constraints + WHERE table_schema = 'latitude' + AND table_name = 'events' + AND constraint_type = 'PRIMARY KEY'; + 2. Uncomment code below and paste pk name manually + + Hope to release this update as soon as possible +*/ + +-- ALTER TABLE "events" DROP CONSTRAINT "";--> statement-breakpoint +ALTER TABLE "latitude"."events" ADD COLUMN "workspace_id" bigint;--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "latitude"."events" ADD CONSTRAINT "events_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "latitude"."workspaces"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "event_workspace_idx" ON "latitude"."events" USING btree ("workspace_id"); \ No newline at end of file diff --git a/packages/core/drizzle/meta/0060_snapshot.json b/packages/core/drizzle/meta/0060_snapshot.json new file mode 100644 index 000000000..318dccc0f --- /dev/null +++ b/packages/core/drizzle/meta/0060_snapshot.json @@ -0,0 +1,2402 @@ +{ + "id": "31261452-484a-43fa-a439-a7f6901487ac", + "prevId": "5f5a2cd9-2dd1-461e-ad9e-f5161b837ef3", + "version": "7", + "dialect": "postgresql", + "tables": { + "latitude.users": { + "name": "users", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "admin": { + "name": "admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + }, + "latitude.sessions": { + "name": "sessions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.workspaces": { + "name": "workspaces", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_creator_id_users_id_fk": { + "name": "workspaces_creator_id_users_id_fk", + "tableFrom": "workspaces", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.memberships": { + "name": "memberships", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invitation_token": { + "name": "invitation_token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_workspace_id_user_id_index": { + "name": "memberships_workspace_id_user_id_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_invitation_token_index": { + "name": "memberships_invitation_token_index", + "columns": [ + { + "expression": "invitation_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_workspace_id_workspaces_id_fk": { + "name": "memberships_workspace_id_workspaces_id_fk", + "tableFrom": "memberships", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "memberships_invitation_token_unique": { + "name": "memberships_invitation_token_unique", + "nullsNotDistinct": false, + "columns": [ + "invitation_token" + ] + } + } + }, + "latitude.api_keys": { + "name": "api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_id_idx": { + "name": "workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_keys_workspace_id_workspaces_id_fk": { + "name": "api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_token_unique": { + "name": "api_keys_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.projects": { + "name": "projects", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_idx": { + "name": "workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "projects_workspace_id_workspaces_id_fk": { + "name": "projects_workspace_id_workspaces_id_fk", + "tableFrom": "projects", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.commits": { + "name": "commits", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "project_commit_order_idx": { + "name": "project_commit_order_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_commit_version": { + "name": "unique_commit_version", + "columns": [ + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_idx": { + "name": "user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "merged_at_idx": { + "name": "merged_at_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "project_id_idx": { + "name": "project_id_idx", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "commits_project_id_projects_id_fk": { + "name": "commits_project_id_projects_id_fk", + "tableFrom": "commits", + "tableTo": "projects", + "schemaTo": "latitude", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "commits_user_id_users_id_fk": { + "name": "commits_user_id_users_id_fk", + "tableFrom": "commits", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "commits_uuid_unique": { + "name": "commits_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.document_versions": { + "name": "document_versions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_versions_unique_document_uuid_commit_id": { + "name": "document_versions_unique_document_uuid_commit_id", + "columns": [ + { + "expression": "document_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_unique_path_commit_id_deleted_at": { + "name": "document_versions_unique_path_commit_id_deleted_at", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_commit_id_idx": { + "name": "document_versions_commit_id_idx", + "columns": [ + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_deleted_at_idx": { + "name": "document_versions_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_path_idx": { + "name": "document_versions_path_idx", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_versions_commit_id_commits_id_fk": { + "name": "document_versions_commit_id_commits_id_fk", + "tableFrom": "document_versions", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.provider_api_keys": { + "name": "provider_api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "provider_apikeys_workspace_id_idx": { + "name": "provider_apikeys_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_name_idx": { + "name": "provider_apikeys_name_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_user_id_idx": { + "name": "provider_apikeys_user_id_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_token_provider_unique": { + "name": "provider_apikeys_token_provider_unique", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_api_keys_author_id_users_id_fk": { + "name": "provider_api_keys_author_id_users_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_api_keys_workspace_id_workspaces_id_fk": { + "name": "provider_api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.document_logs": { + "name": "document_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parameters": { + "name": "parameters", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "custom_identifier": { + "name": "custom_identifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_log_uuid_idx": { + "name": "document_log_uuid_idx", + "columns": [ + { + "expression": "document_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_logs_commit_id_idx": { + "name": "document_logs_commit_id_idx", + "columns": [ + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_logs_commit_id_commits_id_fk": { + "name": "document_logs_commit_id_commits_id_fk", + "tableFrom": "document_logs", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "document_logs_uuid_unique": { + "name": "document_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.provider_logs": { + "name": "provider_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_log_uuid": { + "name": "document_log_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "messages": { + "name": "messages", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "response_object": { + "name": "response_object", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "response_text": { + "name": "response_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'[]'::json" + }, + "tokens": { + "name": "tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "cost_in_millicents": { + "name": "cost_in_millicents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "apiKeyId": { + "name": "apiKeyId", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "generated_at": { + "name": "generated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_logs_provider_id_provider_api_keys_id_fk": { + "name": "provider_logs_provider_id_provider_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "provider_api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "provider_logs_apiKeyId_api_keys_id_fk": { + "name": "provider_logs_apiKeyId_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "apiKeyId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_logs_uuid_unique": { + "name": "provider_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.datasets": { + "name": "datasets", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "csv_delimiter": { + "name": "csv_delimiter", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_key": { + "name": "file_key", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "file_metadata": { + "name": "file_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_workspace_idx": { + "name": "datasets_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_author_idx": { + "name": "datasets_author_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_workspace_id_name_index": { + "name": "datasets_workspace_id_name_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_workspace_id_workspaces_id_fk": { + "name": "datasets_workspace_id_workspaces_id_fk", + "tableFrom": "datasets", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_author_id_users_id_fk": { + "name": "datasets_author_id_users_id_fk", + "tableFrom": "datasets", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations": { + "name": "evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata_id": { + "name": "metadata_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "metadata_type", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_workspace_idx": { + "name": "evaluation_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluation_metadata_idx": { + "name": "evaluation_metadata_idx", + "columns": [ + { + "expression": "metadata_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "metadata_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluations_workspace_id_workspaces_id_fk": { + "name": "evaluations_workspace_id_workspaces_id_fk", + "tableFrom": "evaluations", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "evaluations_uuid_unique": { + "name": "evaluations_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.llm_as_judge_evaluation_metadatas": { + "name": "llm_as_judge_evaluation_metadatas", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true, + "default": "'llm_as_judge'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template_id": { + "name": "template_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "llm_as_judge_evaluation_metadatas_template_id_idx": { + "name": "llm_as_judge_evaluation_metadatas_template_id_idx", + "columns": [ + { + "expression": "template_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk": { + "name": "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk", + "tableFrom": "llm_as_judge_evaluation_metadatas", + "tableTo": "evaluations_templates", + "schemaTo": "latitude", + "columnsFrom": [ + "template_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.connected_evaluations": { + "name": "connected_evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "live": { + "name": "live", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "connected_evaluations_evaluation_idx": { + "name": "connected_evaluations_evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "connected_evaluations_evaluation_id_evaluations_id_fk": { + "name": "connected_evaluations_evaluation_id_evaluations_id_fk", + "tableFrom": "connected_evaluations", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "connected_evaluations_unique_idx": { + "name": "connected_evaluations_unique_idx", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "evaluation_id" + ] + } + } + }, + "latitude.evaluation_results": { + "name": "evaluation_results", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "document_log_id": { + "name": "document_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider_log_id": { + "name": "provider_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resultable_type": { + "name": "resultable_type", + "type": "evaluation_result_types", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "resultable_id": { + "name": "resultable_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_idx": { + "name": "evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_log_idx": { + "name": "document_log_idx", + "columns": [ + { + "expression": "document_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_log_idx": { + "name": "provider_log_idx", + "columns": [ + { + "expression": "provider_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resultable_idx": { + "name": "resultable_idx", + "columns": [ + { + "expression": "resultable_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resultable_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluation_results_evaluation_id_evaluations_id_fk": { + "name": "evaluation_results_evaluation_id_evaluations_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "evaluation_results_document_log_id_document_logs_id_fk": { + "name": "evaluation_results_document_log_id_document_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "document_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "document_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "evaluation_results_provider_log_id_provider_logs_id_fk": { + "name": "evaluation_results_provider_log_id_provider_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "provider_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_templates": { + "name": "evaluations_templates", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "evaluations_templates_category_evaluations_template_categories_id_fk": { + "name": "evaluations_templates_category_evaluations_template_categories_id_fk", + "tableFrom": "evaluations_templates", + "tableTo": "evaluations_template_categories", + "schemaTo": "latitude", + "columnsFrom": [ + "category" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_template_categories": { + "name": "evaluations_template_categories", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.magic_link_tokens": { + "name": "magic_link_tokens", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "expired_at": { + "name": "expired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "magic_link_tokens_user_id_users_id_fk": { + "name": "magic_link_tokens_user_id_users_id_fk", + "tableFrom": "magic_link_tokens", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "magic_link_tokens_token_unique": { + "name": "magic_link_tokens_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.events": { + "name": "events", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "event_workspace_idx": { + "name": "event_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "event_type_idx": { + "name": "event_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "events_workspace_id_workspaces_id_fk": { + "name": "events_workspace_id_workspaces_id_fk", + "tableFrom": "events", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_numbers": { + "name": "evaluation_resultable_numbers", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_texts": { + "name": "evaluation_resultable_texts", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_booleans": { + "name": "evaluation_resultable_booleans", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "latitude.provider": { + "name": "provider", + "schema": "latitude", + "values": [ + "openai", + "anthropic", + "groq", + "mistral", + "azure", + "google" + ] + }, + "latitude.log_source": { + "name": "log_source", + "schema": "latitude", + "values": [ + "playground", + "api", + "evaluation" + ] + }, + "latitude.metadata_type": { + "name": "metadata_type", + "schema": "latitude", + "values": [ + "llm_as_judge" + ] + }, + "public.evaluation_result_types": { + "name": "evaluation_result_types", + "schema": "public", + "values": [ + "evaluation_resultable_booleans", + "evaluation_resultable_texts", + "evaluation_resultable_numbers" + ] + } + }, + "schemas": { + "latitude": "latitude" + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/core/drizzle/meta/_journal.json b/packages/core/drizzle/meta/_journal.json index 14b332001..a97326c0f 100644 --- a/packages/core/drizzle/meta/_journal.json +++ b/packages/core/drizzle/meta/_journal.json @@ -421,6 +421,13 @@ "when": 1727298596943, "tag": "0059_faulty_barracuda", "breakpoints": true + }, + { + "idx": 60, + "version": "7", + "when": 1727708403437, + "tag": "0060_omniscient_gunslinger", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/core/package.json b/packages/core/package.json index 86d3a68bd..325152048 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -68,6 +68,7 @@ "lodash-es": "^4.17.21", "pg": "^8.12.0", "pg-transactional-tests": "^1.0.9", + "posthog-node": "^4.2.0", "prettier": "^3.3.3", "react": "^18.3.1", "socket.io-client": "^4.7.5", @@ -101,6 +102,7 @@ "json-schema": "^0.4.0", "lodash-es": "^4.17.21", "pg": "^8.12.0", + "posthog-node": "^4.2.0", "react": "^18.3.1", "socket.io-client": "^4.7.5", "svelte": "^4.2.19", diff --git a/packages/core/src/events/handlers/createEvaluationResultJob.test.ts b/packages/core/src/events/handlers/createEvaluationResultJob.test.ts index f4e9c5fe8..fe23b4cfb 100644 --- a/packages/core/src/events/handlers/createEvaluationResultJob.test.ts +++ b/packages/core/src/events/handlers/createEvaluationResultJob.test.ts @@ -60,11 +60,14 @@ describe('createEvaluationResultJob', () => { commit: draft, path: 'folder1/doc1', content: factories.helpers.createPrompt({ provider }), + workspace, + user, }) documentVersion = doc.documentVersion commit = await mergeCommit(draft).then((r) => r.unwrap()) evaluation = await factories.createLlmAsJudgeEvaluation({ + user, workspace, prompt: factories.helpers.createPrompt({ provider }), configuration: { diff --git a/packages/core/src/events/handlers/index.ts b/packages/core/src/events/handlers/index.ts index 804304f61..abb59befc 100644 --- a/packages/core/src/events/handlers/index.ts +++ b/packages/core/src/events/handlers/index.ts @@ -1,12 +1,16 @@ import { ChainCallResponse, Commit, + Dataset, DocumentLog, + DocumentVersion, + Evaluation, LogSources, MagicLinkToken, Membership, Message, Project, + ProviderApiKey, ProviderLog, User, Workspace, @@ -20,7 +24,7 @@ import { sendMagicLinkJob } from './sendMagicLinkHandler' type LatitudeEventGeneric< U extends keyof typeof EventHandlers, - T extends Record, + T extends { userEmail?: string; workspaceId?: number; [key: string]: any }, > = { type: U data: T @@ -34,12 +38,15 @@ export type EventHandler = ({ export type MagicLinkTokenCreated = LatitudeEventGeneric< 'magicLinkTokenCreated', - MagicLinkToken + MagicLinkToken & { userEmail: string } +> +export type UserCreatedEvent = LatitudeEventGeneric< + 'userCreated', + User & { workspaceId: number; userEmail: string } > -export type UserCreatedEvent = LatitudeEventGeneric<'userCreated', User> export type MembershipCreatedEvent = LatitudeEventGeneric< 'membershipCreated', - Membership & { authorId?: string } + Membership & { authorId?: string; userEmail?: string } > export type EvaluationRunEvent = LatitudeEventGeneric< 'evaluationRun', @@ -49,6 +56,7 @@ export type EvaluationRunEvent = LatitudeEventGeneric< documentLogUuid: string providerLogUuid: string response: ChainCallResponse + workspaceId: number } > export type DocumentRunEvent = LatitudeEventGeneric< @@ -103,6 +111,8 @@ export type WorkspaceCreatedEvent = LatitudeEventGeneric< { workspace: Workspace user: User + userEmail: string + workspaceId: number } > @@ -111,6 +121,17 @@ export type ProjectCreatedEvent = LatitudeEventGeneric< { project: Project commit: Commit + userEmail: string + workspaceId: number + } +> + +export type CommitCreatedEvent = LatitudeEventGeneric< + 'commitCreated', + { + commit: Commit + userEmail: string + workspaceId: number } > @@ -119,6 +140,79 @@ export type DocumentLogCreatedEvent = LatitudeEventGeneric< DocumentLog > +export type EvaluationCreatedEvent = LatitudeEventGeneric< + 'evaluationCreated', + { + evaluation: Evaluation + userEmail: string + workspaceId: number + } +> + +export type DatasetCreatedEvent = LatitudeEventGeneric< + 'datasetCreated', + { + dataset: Dataset + userEmail: string + workspaceId: number + } +> + +export type ProviderApiKeyCreatedEvent = LatitudeEventGeneric< + 'providerApiKeyCreated', + { + providerApiKey: ProviderApiKey + userEmail: string + workspaceId: number + } +> + +export type UserInvitedEvent = LatitudeEventGeneric< + 'userInvited', + { + invited: User + invitee: User + userEmail: string + workspaceId: number + } +> + +export type EvaluationsConnectedEvent = LatitudeEventGeneric< + 'evaluationsConnected', + { + evaluations: Partial[] // it includes the basic stuff + userEmail: string + workspaceId: number + } +> + +export type CommitPublishedEvent = LatitudeEventGeneric< + 'commitPublished', + { + commit: Commit + userEmail: string + workspaceId: number + } +> + +export type BatchEvaluationRunEvent = LatitudeEventGeneric< + 'batchEvaluationRun', + { + evaluationId: number + workspaceId: number + userEmail: string + } +> + +export type DocumentCreatedEvent = LatitudeEventGeneric< + 'documentCreated', + { + document: DocumentVersion + workspaceId: number + userEmail: string + } +> + export type LatitudeEvent = | MembershipCreatedEvent | UserCreatedEvent @@ -130,7 +224,15 @@ export type LatitudeEvent = | WorkspaceCreatedEvent | ProjectCreatedEvent | DocumentLogCreatedEvent - + | EvaluationCreatedEvent + | DatasetCreatedEvent + | ProviderApiKeyCreatedEvent + | UserInvitedEvent + | CommitCreatedEvent + | CommitPublishedEvent + | EvaluationsConnectedEvent + | BatchEvaluationRunEvent + | DocumentCreatedEvent export interface IEventsHandlers { magicLinkTokenCreated: EventHandler[] membershipCreated: EventHandler[] @@ -142,17 +244,35 @@ export interface IEventsHandlers { workspaceCreated: EventHandler[] projectCreated: EventHandler[] documentLogCreated: EventHandler[] + evaluationCreated: EventHandler[] + datasetCreated: EventHandler[] + providerApiKeyCreated: EventHandler[] + userInvited: EventHandler[] + commitCreated: EventHandler[] + commitPublished: EventHandler[] + evaluationsConnected: EventHandler[] + batchEvaluationRun: EventHandler[] + documentCreated: EventHandler[] } export const EventHandlers: IEventsHandlers = { + evaluationCreated: [], + aiProviderCallCompleted: [], + documentLogCreated: [runLiveEvaluationsJob], + documentRun: [createDocumentLogJob], + evaluationRun: [createEvaluationResultJob], magicLinkTokenCreated: [sendMagicLinkJob], membershipCreated: [sendInvitationToUserJob], - userCreated: [], - evaluationRun: [createEvaluationResultJob], - documentRun: [createDocumentLogJob], + projectCreated: [], providerLogCreated: [], - aiProviderCallCompleted: [], + userCreated: [], workspaceCreated: [], - projectCreated: [], - documentLogCreated: [runLiveEvaluationsJob], + datasetCreated: [], + providerApiKeyCreated: [], + userInvited: [], + commitCreated: [], + commitPublished: [], + evaluationsConnected: [], + batchEvaluationRun: [], + documentCreated: [], } as const diff --git a/packages/core/src/lib/pagination/paginate.test.ts b/packages/core/src/lib/pagination/paginate.test.ts index d79f8a404..3b15a3896 100644 --- a/packages/core/src/lib/pagination/paginate.test.ts +++ b/packages/core/src/lib/pagination/paginate.test.ts @@ -6,6 +6,7 @@ import { users } from '../../schema' import { paginateQuery } from './paginate' const pageUrl = { base: 'http://localhost/my-page' } + describe('paginateQuery', () => { beforeEach(async () => { await Promise.all( @@ -15,7 +16,7 @@ describe('paginateQuery', () => { ) }) - it('returns first 2 users', async () => { + it.only('returns first 2 users', async () => { const { rows, pagination } = await paginateQuery({ dynamicQuery: database .select({ email: users.email }) diff --git a/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts b/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts index 7f2181cde..9342d1973 100644 --- a/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts +++ b/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts @@ -7,6 +7,8 @@ import { ModifiedDocumentType, Project, Providers, + User, + Workspace, } from '../../browser' import { destroyDocument, updateDocument } from '../../services/documents' import { @@ -21,12 +23,16 @@ let commit: Commit let draftCommit: Commit let documents: Record let repo: CommitsRepository +let workspace: Workspace +let user: User + describe('publishDraftCommit', () => { beforeEach(async () => { const { project: prj, commit: cmt, - user, + user: usr, + workspace: ws, documents: docs, } = await createProject({ providers: [{ type: Providers.OpenAI, name: 'openai' }], @@ -40,6 +46,8 @@ describe('publishDraftCommit', () => { doc2: helpers.createPrompt({ provider: 'openai', content: 'content2' }), }, }) + workspace = ws + user = usr project = prj commit = cmt const { commit: draft } = await createDraft({ project, user }) @@ -90,6 +98,8 @@ describe('publishDraftCommit', () => { it('show created documents', async () => { const { documentVersion: newDoc } = await createDocumentVersion({ + workspace: workspace, + user: user, commit: draftCommit, path: 'folder1/doc3', content: helpers.createPrompt({ diff --git a/packages/core/src/repositories/commitsRepository/index.test.ts b/packages/core/src/repositories/commitsRepository/index.test.ts index fac511be8..a63d13a6f 100644 --- a/packages/core/src/repositories/commitsRepository/index.test.ts +++ b/packages/core/src/repositories/commitsRepository/index.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from 'vitest' -import type { Project, ProviderApiKey, User } from '../../browser' +import type { Project, ProviderApiKey, User, Workspace } from '../../browser' import { CommitStatus } from '../../constants' import { mergeCommit } from '../../services/commits' import { createNewDocument } from '../../services/documents' @@ -8,14 +8,17 @@ import * as factories from '../../tests/factories' import { CommitsRepository } from './index' async function createDraftsCommits( - project: Project, user: User, + workpace: Workspace, + project: Project, provider: ProviderApiKey, ) { const results = [] for (let i = 0; i < 10; i++) { const draft = await factories.createDraft({ project, user }) await createNewDocument({ + user, + workspace: workpace, commit: draft.commit, path: `${i}/foo`, content: factories.helpers.createPrompt({ provider }), @@ -31,13 +34,15 @@ let repository: CommitsRepository describe('Commits by project', () => { beforeEach(async () => { let { + workspace, project: firstProject, user, providers, } = await factories.createProject() const drafsCommits = await createDraftsCommits( - firstProject, user, + workspace, + firstProject, providers[0]!, ) await Promise.all([ @@ -73,9 +78,10 @@ describe('Commits by project', () => { describe('findAll', () => { beforeEach(async () => { - const { project, user, providers } = await factories.createProject() + const { project, workspace, user, providers } = + await factories.createProject() - await createDraftsCommits(project, user, providers[0]!) + await createDraftsCommits(user, workspace, project, providers[0]!) }) it('does not return commits from other workspaces', async () => { diff --git a/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts b/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts index 6379b116c..9593dffef 100644 --- a/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts +++ b/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts @@ -64,6 +64,7 @@ describe('getConnectedDocumentsWithMetadata', () => { await Promise.all( (documentsArr ?? documents).map(async (document) => { await connectEvaluations({ + user, workspace, documentUuid: document.documentUuid, evaluationUuids: [evaluation.uuid], @@ -160,6 +161,8 @@ describe('getConnectedDocumentsWithMetadata', () => { it('does not return documents that only exist in a draft, even when its connected to an evaluation', async () => { const { commit: draft } = await factories.createDraft({ project, user }) const draftDocument = await createNewDocument({ + user, + workspace, commit: draft, path: 'foo', content: documentContent('New document'), diff --git a/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts b/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts index bc0c79406..ce9099c8e 100644 --- a/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts +++ b/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts @@ -8,9 +8,12 @@ import { DocumentVersionsRepository } from './index' describe('getDocumentAtCommit', () => { it('return doc from merged commit', async () => { - const { project, user, providers } = await factories.createProject() + const { workspace, project, user, providers } = + await factories.createProject() const { commit } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ + workspace, + user, commit: commit, path: 'folder1/doc1', content: factories.helpers.createPrompt({ diff --git a/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts b/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts index 09e51de17..ca6565f62 100644 --- a/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts +++ b/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts @@ -70,7 +70,8 @@ describe('getDocumentsAtCommit', () => { describe('documents for each commit', () => { beforeEach(async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() const documentsScope = new DocumentVersionsRepository(project.workspaceId) const { commit: commit1 } = await factories.createDraft({ project, user }) const { commit: commit2 } = await factories.createDraft({ project, user }) @@ -78,6 +79,8 @@ describe('getDocumentsAtCommit', () => { // Initial document const { documentVersion: doc1 } = await factories.createDocumentVersion({ + workspace, + user, commit: commit1, path: 'doc1', content: ctx.factories.helpers.createPrompt({ @@ -174,12 +177,15 @@ describe('getDocumentsAtCommit', () => { describe('documents from previous commits', () => { beforeEach(async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() const documentsScope = new DocumentVersionsRepository(project.workspaceId) // Doc 1 const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc1 } = await factories.createDocumentVersion({ + workspace, + user, commit: commit1, path: 'doc1', content: ctx.factories.helpers.createPrompt({ @@ -192,6 +198,8 @@ describe('getDocumentsAtCommit', () => { // Doc 2 const { commit: commit2 } = await factories.createDraft({ project, user }) const { documentVersion: doc2 } = await factories.createDocumentVersion({ + workspace, + user, commit: commit2, path: 'doc2', content: ctx.factories.helpers.createPrompt({ diff --git a/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts b/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts index aa18627bf..ee43ee509 100644 --- a/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts +++ b/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts @@ -10,6 +10,7 @@ describe('findEvaluationResultsByDocumentUuid', () => { const { workspace, project, user, providers } = await factories.createProject() const evaluation = await factories.createLlmAsJudgeEvaluation({ + user, workspace, prompt: factories.helpers.createPrompt({ provider: providers[0]! }), configuration: { @@ -22,6 +23,8 @@ describe('findEvaluationResultsByDocumentUuid', () => { const { commit: draft } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'folder1/doc1', content: factories.helpers.createPrompt({ provider: providers[0]! }), diff --git a/packages/core/src/repositories/evaluationResultsRepository/index.test.ts b/packages/core/src/repositories/evaluationResultsRepository/index.test.ts index c62462c0d..2312432bb 100644 --- a/packages/core/src/repositories/evaluationResultsRepository/index.test.ts +++ b/packages/core/src/repositories/evaluationResultsRepository/index.test.ts @@ -20,6 +20,7 @@ describe('EvaluationResultsRepository', () => { documents: [document1], commit: commit1, providers: [provider1], + user: user1, } = await createProject({ providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { @@ -30,6 +31,7 @@ describe('EvaluationResultsRepository', () => { }) const { workspace: workspace2, + user: user2, documents: [document2], commit: commit2, providers: [provider2], @@ -44,9 +46,11 @@ describe('EvaluationResultsRepository', () => { // Create evaluations for each workspace const evaluation1 = await createLlmAsJudgeEvaluation({ + user: user1, workspace: workspace1, }) const evaluation2 = await createLlmAsJudgeEvaluation({ + user: user2, workspace: workspace2, }) diff --git a/packages/core/src/schema/models/events.ts b/packages/core/src/schema/models/events.ts index 7e3ebcfb0..ffdae6639 100644 --- a/packages/core/src/schema/models/events.ts +++ b/packages/core/src/schema/models/events.ts @@ -1,30 +1,22 @@ -import { bigserial, index, jsonb, varchar } from 'drizzle-orm/pg-core' +import { bigint, bigserial, index, jsonb, varchar } from 'drizzle-orm/pg-core' import { latitudeSchema } from '../db-schema' import { timestamps } from '../schemaHelpers' +import { workspaces } from './workspaces' export const events = latitudeSchema.table( 'events', { - id: bigserial('id', { mode: 'number' }).notNull().primaryKey(), - // Dear developer, - // - // Ideally we want to scope events by workspace but this requires some - // extra work that we are not willing to pay right now. Mainly, we need to - // add workspaceId to all our existing events. Uncomment the following line - // once you are ready to pay this price. - // - // TODO: Uncomment when we are ready - // workspaceId: bigint('workspace_id', { mode: 'number' }) - // .notNull() - // .references(() => workspaces.id, { onDelete: 'cascade' }), + id: bigserial('id', { mode: 'number' }), + workspaceId: bigint('workspace_id', { mode: 'number' }).references( + () => workspaces.id, + ), type: varchar('type', { length: 256 }).notNull(), data: jsonb('data').notNull(), ...timestamps(), }, (table) => ({ - // TODO: Uncomment when we are ready - // eventWorkspaceIdx: index('event_workspace_idx').on(table.workspaceId), + eventWorkspaceIdx: index('event_workspace_idx').on(table.workspaceId), eventTypeIdx: index('event_type_idx').on(table.type), }), ) diff --git a/packages/core/src/services/commits/create.ts b/packages/core/src/services/commits/create.ts index 4cd2c2718..29016118d 100644 --- a/packages/core/src/services/commits/create.ts +++ b/packages/core/src/services/commits/create.ts @@ -1,5 +1,6 @@ import { Commit, Project, User } from '../../browser' import { database, Database } from '../../client' +import { publisher } from '../../events/publisher' import { Result, Transaction } from '../../lib' import { commits } from '../../schema' @@ -33,6 +34,15 @@ export async function createCommit({ .returning() const createdCommit = result[0] + publisher.publishLater({ + type: 'commitCreated', + data: { + commit: createdCommit!, + userEmail: user.email, + workspaceId: project.workspaceId, + }, + }) + return Result.ok(createdCommit!) }, db) } diff --git a/packages/core/src/services/commits/merge.test.ts b/packages/core/src/services/commits/merge.test.ts index ba9d14d97..cc08b8559 100644 --- a/packages/core/src/services/commits/merge.test.ts +++ b/packages/core/src/services/commits/merge.test.ts @@ -10,10 +10,13 @@ import { mergeCommit } from './merge' describe('mergeCommit', () => { it('merges a commit', async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, workspace, user, providers } = + await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ + user, + workspace, commit, path: 'foo', content: ctx.factories.helpers.createPrompt({ provider: providers[0]! }), @@ -36,10 +39,13 @@ describe('mergeCommit', () => { }) it('recomputes all changes in the commit', async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ + user, + workspace, commit, path: 'foo', content: ctx.factories.helpers.createPrompt({ provider: providers[0]! }), @@ -65,10 +71,13 @@ describe('mergeCommit', () => { }) it('fails when trying to merge a commit with syntax errors', async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ + user, + workspace, commit, path: 'foo', content: ctx.factories.helpers.createPrompt({ @@ -122,13 +131,16 @@ describe('mergeCommit', () => { }) it('increases the version number of the commit', async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() const { commit: commit1 } = await ctx.factories.createDraft({ project, user, }) const doc = await createNewDocument({ + workspace, + user, commit: commit1, path: 'foo1', content: ctx.factories.helpers.createPrompt({ diff --git a/packages/core/src/services/datasets/create.ts b/packages/core/src/services/datasets/create.ts index 5381086c2..8c4088b8a 100644 --- a/packages/core/src/services/datasets/create.ts +++ b/packages/core/src/services/datasets/create.ts @@ -4,6 +4,7 @@ import slugify from '@sindresorhus/slugify' import { User, Workspace } from '../../browser' import { database } from '../../client' +import { publisher } from '../../events/publisher' import { Result, Transaction } from '../../lib' import { diskFactory, DiskWrapper } from '../../lib/disk' import { syncReadCsv } from '../../lib/readCsv' @@ -63,6 +64,18 @@ export const createDataset = async ( const dataset = inserts[0]! + publisher.publishLater({ + type: 'datasetCreated', + data: { + dataset: { + ...dataset, + author, + }, + workspaceId: workspace.id, + userEmail: author.email, + }, + }) + return Result.ok({ ...dataset, author: { diff --git a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts index 86a02dafd..92e778c80 100644 --- a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts +++ b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts @@ -18,6 +18,8 @@ describe('computeDocumentLogWithMetadata', () => { user: setup.user, }) const { documentVersion } = await factories.createDocumentVersion({ + workspace: setup.workspace, + user: setup.user, commit, path: 'folder1/doc1', content: factories.helpers.createPrompt({ provider, content: 'Doc 1' }), diff --git a/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts index ff54b1652..af5c7a5fc 100644 --- a/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts +++ b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.test.ts @@ -7,9 +7,12 @@ import { computeDocumentLogsWithMetadataQuery } from './computeDocumentLogsWithM describe('getDocumentLogsWithMetadata', () => { it('return all logs from merged commits', async () => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ + workspace, + user, commit: commit1, path: 'folder1/doc1', content: factories.helpers.createPrompt({ @@ -50,9 +53,12 @@ describe('getDocumentLogsWithMetadata', () => { }) it('includes logs from specified draft', async () => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ + workspace, + user, commit: commit1, path: 'folder1/doc1', content: factories.helpers.createPrompt({ @@ -109,9 +115,12 @@ describe('getDocumentLogsWithMetadata', () => { }) it('does not include logs from non-specified drafts', async () => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ + workspace, + user, commit: commit1, path: 'folder1/doc1', content: factories.helpers.createPrompt({ @@ -166,9 +175,12 @@ describe('getDocumentLogsWithMetadata', () => { }) it('returns a sum of tokens and cost', async () => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ + workspace, + user, commit, path: 'folder1/doc1', content: factories.helpers.createPrompt({ diff --git a/packages/core/src/services/documents/create.test.ts b/packages/core/src/services/documents/create.test.ts index 42e11188e..df2ddf132 100644 --- a/packages/core/src/services/documents/create.test.ts +++ b/packages/core/src/services/documents/create.test.ts @@ -6,10 +6,12 @@ import { createNewDocument } from './create' describe('createNewDocument', () => { it('creates a new document version in the commit', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, workspace } = await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) const documentResult = await createNewDocument({ + workspace, + user, commit, path: 'foo', }) @@ -25,15 +27,19 @@ describe('createNewDocument', () => { }) it('fails if there is another document with the same path', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, workspace } = await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ + workspace, + user, commit, path: 'foo', }) const result = await createNewDocument({ + workspace, + user, commit, path: 'foo', }) @@ -45,9 +51,12 @@ describe('createNewDocument', () => { }) it('fails when trying to create a document in a merged commit', async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() let { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ + workspace, + user, commit, path: 'foo', content: ctx.factories.helpers.createPrompt({ provider: providers[0]! }), @@ -55,6 +64,8 @@ describe('createNewDocument', () => { commit = await mergeCommit(commit).then((r) => r.unwrap()) const result = await createNewDocument({ + workspace, + user, commit, path: 'foo', }) @@ -64,10 +75,13 @@ describe('createNewDocument', () => { }) it('creates a new document with default content when no content is provided', async (ctx) => { - const { project, user, providers } = await ctx.factories.createProject() + const { project, user, workspace, providers } = + await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) const documentResult = await createNewDocument({ + workspace, + user, commit, path: 'newdoc', }) @@ -95,12 +109,14 @@ model: gpt-4o-mini }) it('creates the document without the frontmatter if no provider is found', async (ctx) => { - const { project, user } = await ctx.factories.createProject({ + const { project, user, workspace } = await ctx.factories.createProject({ providers: [], }) const { commit } = await ctx.factories.createDraft({ project, user }) const result = await createNewDocument({ + workspace, + user, commit, path: 'newdoc', }) diff --git a/packages/core/src/services/documents/create.ts b/packages/core/src/services/documents/create.ts index b153ca65e..5d7d8cbcc 100644 --- a/packages/core/src/services/documents/create.ts +++ b/packages/core/src/services/documents/create.ts @@ -2,11 +2,13 @@ import { eq } from 'drizzle-orm' import { findFirstModelForProvider, + User, + Workspace, type Commit, type DocumentVersion, } from '../../browser' import { database } from '../../client' -import { findWorkspaceFromCommit } from '../../data-access' +import { publisher } from '../../events/publisher' import { Result, Transaction, TypedResult } from '../../lib' import { BadRequestError } from '../../lib/errors' import { @@ -17,10 +19,14 @@ import { documentVersions } from '../../schema' export async function createNewDocument( { + workspace, + user, commit, path, content, }: { + workspace: Workspace + user: User commit: Commit path: string content?: string @@ -32,7 +38,6 @@ export async function createNewDocument( return Result.error(new BadRequestError('Cannot modify a merged commit')) } - const workspace = await findWorkspaceFromCommit(commit, tx) const docsScope = new DocumentVersionsRepository(workspace!.id, tx) const currentDocs = await docsScope @@ -71,6 +76,15 @@ model: ${findFirstModelForProvider(provider.provider)} .set({ resolvedContent: null }) .where(eq(documentVersions.commitId, commit.id)) + publisher.publishLater({ + type: 'documentCreated', + data: { + document: newDoc[0]!, + workspaceId: workspace.id, + userEmail: user.email, + }, + }) + return Result.ok(newDoc[0]!) }, db) } diff --git a/packages/core/src/services/documents/destroyFolder.test.ts b/packages/core/src/services/documents/destroyFolder.test.ts index 60cf50602..cff182518 100644 --- a/packages/core/src/services/documents/destroyFolder.test.ts +++ b/packages/core/src/services/documents/destroyFolder.test.ts @@ -25,9 +25,12 @@ describe('removing folders', () => { }) it('throws error if commit is merged', async (ctx) => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit: draft } = await factories.createDraft({ project, user }) await createNewDocument({ + workspace, + user, commit: draft, path: 'foo', content: ctx.factories.helpers.createPrompt({ @@ -47,9 +50,12 @@ describe('removing folders', () => { }) it('destroy folder that were in draft document but not in previous merged commits', async (ctx) => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit: draft } = await factories.createDraft({ project, user }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'root-folder/some-folder/doc1', content: ctx.factories.helpers.createPrompt({ @@ -58,6 +64,8 @@ describe('removing folders', () => { }), }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'root-folder/some-folder/doc2', content: ctx.factories.helpers.createPrompt({ @@ -66,6 +74,8 @@ describe('removing folders', () => { }), }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'root-folder/some-folder/inner-folder/doc42', content: ctx.factories.helpers.createPrompt({ @@ -74,6 +84,8 @@ describe('removing folders', () => { }), }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'root-folder/other-nested-folder/doc3', content: ctx.factories.helpers.createPrompt({ @@ -82,6 +94,8 @@ describe('removing folders', () => { }), }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'root-folder/some-foldernoisadoc', content: ctx.factories.helpers.createPrompt({ @@ -90,6 +104,8 @@ describe('removing folders', () => { }), }) await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'other-foler/doc4', content: ctx.factories.helpers.createPrompt({ diff --git a/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts b/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts index 66708775f..4c1f4d55b 100644 --- a/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts +++ b/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts @@ -10,10 +10,13 @@ import { updateDocument } from './update' describe('destroyOrSoftDeleteDocuments', () => { it('remove documents that were not present in merged commits', async (ctx) => { - const { project, user, providers } = await factories.createProject() + const { project, user, workspace, providers } = + await factories.createProject() const { commit: draft } = await factories.createDraft({ project, user }) const { documentVersion: draftDocument } = await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'doc1', content: ctx.factories.helpers.createPrompt({ diff --git a/packages/core/src/services/documents/update.test.ts b/packages/core/src/services/documents/update.test.ts index 5f53eade6..dbac44827 100644 --- a/packages/core/src/services/documents/update.test.ts +++ b/packages/core/src/services/documents/update.test.ts @@ -55,6 +55,8 @@ describe('updateDocument', () => { const docsScope = new DocumentVersionsRepository(project.workspaceId) const { commit } = await ctx.factories.createDraft({ project, user }) const { documentVersion: doc } = await ctx.factories.createDocumentVersion({ + workspace, + user, commit: commit, path: 'doc1', content: ctx.factories.helpers.createPrompt({ diff --git a/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts b/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts index 28a6dc431..206b90e2f 100644 --- a/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts +++ b/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts @@ -35,6 +35,8 @@ describe('evaluation results aggregations', () => { const { commit: draft } = await factories.createDraft({ project, user }) const doc = await factories.createDocumentVersion({ + workspace, + user, commit: draft, path: 'folder1/doc1', content: factories.helpers.createPrompt({ provider }), @@ -47,6 +49,7 @@ describe('evaluation results aggregations', () => { describe('numeric evaluations', () => { beforeEach(async () => { evaluation = await factories.createLlmAsJudgeEvaluation({ + user, workspace, prompt: factories.helpers.createPrompt({ provider }), configuration: { @@ -130,6 +133,7 @@ describe('evaluation results aggregations', () => { describe('text evaluations', () => { beforeEach(async () => { evaluation = await factories.createLlmAsJudgeEvaluation({ + user, workspace, prompt: factories.helpers.createPrompt({ provider }), configuration: { diff --git a/packages/core/src/services/evaluationResults/create.test.ts b/packages/core/src/services/evaluationResults/create.test.ts index caa5f4e1f..14c79f10d 100644 --- a/packages/core/src/services/evaluationResults/create.test.ts +++ b/packages/core/src/services/evaluationResults/create.test.ts @@ -19,6 +19,7 @@ async function setupTest(configurationType: EvaluationResultableType) { const provider = providers[0] const evaluation = await factories.createLlmAsJudgeEvaluation({ workspace, + user, configuration: { type: configurationType, ...(configurationType === EvaluationResultableType.Number @@ -28,6 +29,8 @@ async function setupTest(configurationType: EvaluationResultableType) { }) const { commit } = await factories.createDraft({ project, user }) const { documentVersion } = await factories.createDocumentVersion({ + workspace, + user, commit, path: 'folder1/doc1', content: ` diff --git a/packages/core/src/services/evaluations/connect.ts b/packages/core/src/services/evaluations/connect.ts index b5503ded2..49300864f 100644 --- a/packages/core/src/services/evaluations/connect.ts +++ b/packages/core/src/services/evaluations/connect.ts @@ -1,5 +1,6 @@ -import { ConnectedEvaluation, Workspace } from '../../browser' +import { ConnectedEvaluation, User, Workspace } from '../../browser' import { database } from '../../client' +import { publisher } from '../../events/publisher' import { NotFoundError, PromisedResult, @@ -20,11 +21,13 @@ export function connectEvaluations( documentUuid, evaluationUuids, templateIds, + user, }: { workspace: Workspace documentUuid: string evaluationUuids?: string[] templateIds?: number[] + user: User }, db = database, ): PromisedResult { @@ -46,7 +49,7 @@ export function connectEvaluations( // evaluations service. const importedEvaluations = await Promise.all( templateIds?.map((templateId) => - importLlmAsJudgeEvaluation({ workspace, templateId }, tx), + importLlmAsJudgeEvaluation({ workspace, user, templateId }, tx), ) ?? [], ) @@ -86,6 +89,15 @@ export function connectEvaluations( ) .returning() + publisher.publishLater({ + type: 'evaluationsConnected', + data: { + evaluations: rezults, + userEmail: user.email, + workspaceId: workspace.id, + }, + }) + return Result.ok(rezults) }, db, diff --git a/packages/core/src/services/evaluations/create.test.ts b/packages/core/src/services/evaluations/create.test.ts index 1f5df8e26..06992f166 100644 --- a/packages/core/src/services/evaluations/create.test.ts +++ b/packages/core/src/services/evaluations/create.test.ts @@ -25,6 +25,7 @@ describe('createEvaluation', () => { const description = 'Test Description' const metadata = { prompt: 'Test prompt' } const result = await createEvaluation({ + user, workspace, name, description, @@ -55,6 +56,7 @@ describe('createEvaluation', () => { const metadata = { prompt: 'Test prompt' } const result = await createEvaluation({ workspace, + user, name, description, type: EvaluationMetadataType.LlmAsJudge, @@ -94,6 +96,7 @@ ${metadata.prompt} const result = await createEvaluation({ workspace, + user, name, description, configuration: { @@ -115,6 +118,7 @@ ${metadata.prompt} const result = await createEvaluation({ workspace, + user, name, description, type: EvaluationMetadataType.LlmAsJudge, @@ -136,6 +140,7 @@ ${metadata.prompt} it('returns an error for invalid evaluation type', async () => { const result = await createEvaluation({ workspace, + user, name: 'Test Evaluation', description: 'Test Description', type: 'InvalidType' as EvaluationMetadataType, @@ -163,6 +168,7 @@ ${metadata.prompt} const result = await createEvaluation({ workspace, + user, name: 'Test Evaluation', description: 'Test Description', type: EvaluationMetadataType.LlmAsJudge, diff --git a/packages/core/src/services/evaluations/create.ts b/packages/core/src/services/evaluations/create.ts index dc83ede8a..f983929d9 100644 --- a/packages/core/src/services/evaluations/create.ts +++ b/packages/core/src/services/evaluations/create.ts @@ -2,10 +2,12 @@ import { EvaluationMetadataType, EvaluationResultConfiguration, findFirstModelForProvider, + User, Workspace, } from '../../browser' import { database } from '../../client' import { findEvaluationTemplateById } from '../../data-access' +import { publisher } from '../../events/publisher' import { BadRequestError, Result, Transaction } from '../../lib' import { ProviderApiKeysRepository } from '../../repositories' import { evaluations, llmAsJudgeEvaluationMetadatas } from '../../schema' @@ -17,10 +19,19 @@ type Props = { type: EvaluationMetadataType configuration: EvaluationResultConfiguration metadata?: Record + user: User } export async function createEvaluation( - { workspace, name, description, type, configuration, metadata = {} }: Props, + { + workspace, + name, + description, + type, + configuration, + user, + metadata = {}, + }: Props, db = database, ) { const providerScope = new ProviderApiKeysRepository(workspace!.id, db) @@ -66,6 +77,15 @@ ${meta.prompt} ]) .returning() + publisher.publishLater({ + type: 'evaluationCreated', + data: { + evaluation: result[0]!, + workspaceId: workspace.id, + userEmail: user.email, + }, + }) + return Result.ok({ ...result[0]!, metadata: metadataTable[0]!, @@ -74,7 +94,11 @@ ${meta.prompt} } export async function importLlmAsJudgeEvaluation( - { workspace, templateId }: { workspace: Workspace; templateId: number }, + { + workspace, + user, + templateId, + }: { workspace: Workspace; user: User; templateId: number }, db = database, ) { const templateResult = await findEvaluationTemplateById(templateId, db) @@ -83,6 +107,7 @@ export async function importLlmAsJudgeEvaluation( return await createEvaluation( { + user, workspace, name: template.name, description: template.description, diff --git a/packages/core/src/services/evaluations/run.ts b/packages/core/src/services/evaluations/run.ts index d3f5e17b4..2885a4427 100644 --- a/packages/core/src/services/evaluations/run.ts +++ b/packages/core/src/services/evaluations/run.ts @@ -123,6 +123,7 @@ export const runEvaluation = async ( documentLogUuid: documentLog.uuid, providerLogUuid: response.providerLog.uuid, response, + workspaceId: evaluation.workspaceId, }, }) }) diff --git a/packages/core/src/services/magicLinkTokens/create.ts b/packages/core/src/services/magicLinkTokens/create.ts index a3a1f440e..3e1a33c42 100644 --- a/packages/core/src/services/magicLinkTokens/create.ts +++ b/packages/core/src/services/magicLinkTokens/create.ts @@ -16,7 +16,10 @@ export async function createMagicLinkToken( publisher.publishLater({ type: 'magicLinkTokenCreated', - data: magicLinkToken[0]!, + data: { + ...magicLinkToken[0]!, + userEmail: user.email, + }, }) return Result.ok(magicLinkToken[0]!) diff --git a/packages/core/src/services/memberships/create.ts b/packages/core/src/services/memberships/create.ts index f396fcc95..682631ae3 100644 --- a/packages/core/src/services/memberships/create.ts +++ b/packages/core/src/services/memberships/create.ts @@ -45,6 +45,7 @@ const publishEvent = ({ data: { ...membership, authorId: author?.id, + userEmail: author?.email, }, }) } diff --git a/packages/core/src/services/posthog.ts b/packages/core/src/services/posthog.ts new file mode 100644 index 000000000..d449d0055 --- /dev/null +++ b/packages/core/src/services/posthog.ts @@ -0,0 +1,12 @@ +import { env } from '@latitude-data/env' +import { PostHog } from 'posthog-node' + +export function PostHogClient() { + const posthogClient = new PostHog(env.NEXT_PUBLIC_POSTHOG_KEY, { + host: env.NEXT_PUBLIC_POSTHOG_HOST, + flushAt: 1, + flushInterval: 0, + }) + + return posthogClient +} diff --git a/packages/core/src/services/projects/create.ts b/packages/core/src/services/projects/create.ts index 2a21050fc..2ad533cd0 100644 --- a/packages/core/src/services/projects/create.ts +++ b/packages/core/src/services/projects/create.ts @@ -41,7 +41,12 @@ export async function createProject( publisher.publishLater({ type: 'projectCreated', - data: { project, commit: result.value }, + data: { + project, + commit: result.value, + workspaceId: workspace.id, + userEmail: user.email, + }, }) return Result.ok({ project, commit: result.value }) diff --git a/packages/core/src/services/projects/import.ts b/packages/core/src/services/projects/import.ts index bfa387afe..e157902bb 100644 --- a/packages/core/src/services/projects/import.ts +++ b/packages/core/src/services/projects/import.ts @@ -52,6 +52,8 @@ export async function importDefaultProject( defaultDocuments.value.map(async (document) => createNewDocument( { + workspace, + user, commit, path: document.path, content: document.content, diff --git a/packages/core/src/services/providerApiKeys/create.ts b/packages/core/src/services/providerApiKeys/create.ts index 72d0eab91..66b67e76c 100644 --- a/packages/core/src/services/providerApiKeys/create.ts +++ b/packages/core/src/services/providerApiKeys/create.ts @@ -1,17 +1,18 @@ -import { Providers, Workspace } from '../../browser' +import { Providers, User, Workspace } from '../../browser' import { database } from '../../client' +import { publisher } from '../../events/publisher' import { Result, Transaction } from '../../lib' import { providerApiKeys } from '../../schema' export type Props = { - workspace: Partial + workspace: Workspace provider: Providers token: string name: string - authorId: string + author: User } export function createProviderApiKey( - { workspace, provider, token, name, authorId }: Props, + { workspace, provider, token, name, author }: Props, db = database, ) { return Transaction.call(async (tx) => { @@ -22,10 +23,19 @@ export function createProviderApiKey( provider, token, name, - authorId, + authorId: author.id, }) .returning() + publisher.publishLater({ + type: 'providerApiKeyCreated', + data: { + providerApiKey: result[0]!, + workspaceId: workspace.id, + userEmail: author.email, + }, + }) + return Result.ok(result[0]!) }, db) } diff --git a/packages/core/src/services/users/invite.ts b/packages/core/src/services/users/invite.ts index c78a26bac..5180c1781 100644 --- a/packages/core/src/services/users/invite.ts +++ b/packages/core/src/services/users/invite.ts @@ -2,6 +2,7 @@ import { eq } from 'drizzle-orm' import { User, Workspace } from '../../browser' import { database } from '../../client' +import { publisher } from '../../events/publisher' import { Result, Transaction } from '../../lib' import { users } from '../../schema' import { createMembership } from '../memberships/create' @@ -33,6 +34,16 @@ export async function inviteUser( r.unwrap(), ) + publisher.publishLater({ + type: 'userInvited', + data: { + invited: user, + invitee: author, + userEmail: author.email, + workspaceId: workspace.id, + }, + }) + return Result.ok(user) }, db) } diff --git a/packages/core/src/services/workspaces/create.ts b/packages/core/src/services/workspaces/create.ts index f9e9f0c73..708500934 100644 --- a/packages/core/src/services/workspaces/create.ts +++ b/packages/core/src/services/workspaces/create.ts @@ -25,7 +25,12 @@ export async function createWorkspace( publisher.publishLater({ type: 'workspaceCreated', - data: { workspace, user }, + data: { + workspace, + user, + workspaceId: workspace.id, + userEmail: user.email, + }, }) return Result.ok(workspace) diff --git a/packages/core/src/services/workspaces/usage.test.ts b/packages/core/src/services/workspaces/usage.test.ts index 1009f9335..2e0cf83e5 100644 --- a/packages/core/src/services/workspaces/usage.test.ts +++ b/packages/core/src/services/workspaces/usage.test.ts @@ -7,7 +7,7 @@ import { computeWorkspaceUsage, getLatestRenewalDate } from './usage' describe('computeWorkspaceUsage', () => { it('calculates usage correctly when there are evaluation results and document logs', async (ctx) => { - const { workspace, commit, documents, evaluations } = + const { workspace, user, commit, documents, evaluations } = await ctx.factories.createProject({ providers: [{ type: Providers.OpenAI, name: 'test' }], documents: { @@ -24,6 +24,7 @@ describe('computeWorkspaceUsage', () => { const document = documents[0]! const evaluation = evaluations[0]! await connectEvaluations({ + user, workspace, documentUuid: document.documentUuid, evaluationUuids: [evaluation.uuid], @@ -84,13 +85,16 @@ describe('computeWorkspaceUsage', () => { documents: documents2, evaluations: evaluations2, commit: commit2, + user, } = await ctx.factories.createProject({ providers: [{ type: Providers.OpenAI, name: 'bar' }], documents: { bar: ctx.factories.helpers.createPrompt({ provider: 'bar' }), }, evaluations: [ - { prompt: ctx.factories.helpers.createPrompt({ provider: 'bar' }) }, + { + prompt: ctx.factories.helpers.createPrompt({ provider: 'bar' }), + }, ], }) @@ -104,12 +108,14 @@ describe('computeWorkspaceUsage', () => { await connectEvaluations({ workspace: workspace1, + user, documentUuid: document1.documentUuid, evaluationUuids: [evaluation1.uuid], }) await connectEvaluations({ workspace: workspace2, + user, documentUuid: document2.documentUuid, evaluationUuids: [evaluation2.uuid], }) @@ -175,7 +181,7 @@ describe('computeWorkspaceUsage', () => { }) it('calculates usage correctly when there are no evaluation results or document logs', async (ctx) => { - const { workspace, documents, evaluations } = + const { workspace, user, documents, evaluations } = await ctx.factories.createProject({ providers: [{ type: Providers.OpenAI, name: 'test' }], documents: { @@ -190,6 +196,7 @@ describe('computeWorkspaceUsage', () => { const evaluation = evaluations[0]! await connectEvaluations({ workspace, + user, documentUuid: document.documentUuid, evaluationUuids: [evaluation.uuid], }) @@ -204,6 +211,7 @@ describe('computeWorkspaceUsage', () => { it('calculates usage correctly across multiple projects within the workspace', async (ctx) => { const { workspace, + user: user1, commit: commit1, documents: documents1, evaluations, @@ -217,13 +225,16 @@ describe('computeWorkspaceUsage', () => { ], }) - const { commit: commit2, documents: documents2 } = - await ctx.factories.createProject({ - workspace, - documents: { - bar: ctx.factories.helpers.createPrompt({ provider: 'test' }), - }, - }) + const { + commit: commit2, + documents: documents2, + user: user2, + } = await ctx.factories.createProject({ + workspace, + documents: { + bar: ctx.factories.helpers.createPrompt({ provider: 'test' }), + }, + }) const NUM_DOC_LOGS_PER_PROJECT = 5 const NUM_EVAL_LOGS_PER_PROJECT = 5 @@ -234,12 +245,14 @@ describe('computeWorkspaceUsage', () => { await connectEvaluations({ workspace, + user: user1, documentUuid: document1.documentUuid, evaluationUuids: [evaluation.uuid], }) await connectEvaluations({ workspace, + user: user2, documentUuid: document2.documentUuid, evaluationUuids: [evaluation.uuid], }) diff --git a/packages/core/src/tests/factories/connectedEvaluations.ts b/packages/core/src/tests/factories/connectedEvaluations.ts index 6c814f02d..b372462cb 100644 --- a/packages/core/src/tests/factories/connectedEvaluations.ts +++ b/packages/core/src/tests/factories/connectedEvaluations.ts @@ -4,6 +4,7 @@ import { ConnectedEvaluation, EvaluationMetadataType, EvaluationResultableType, + User, Workspace, } from '../../browser' import { @@ -14,14 +15,17 @@ import { export async function createConnectedEvaluation({ workspace, evaluationUuid, + user, documentUuid, }: { workspace: Workspace + user: User evaluationUuid?: string documentUuid: string }): Promise { if (!evaluationUuid) { const evaluation = await createEvaluation({ + user, workspace, name: faker.company.name(), description: faker.lorem.sentence(), @@ -40,6 +44,7 @@ export async function createConnectedEvaluation({ const connectedEvaluations = await connectEvaluations({ workspace, + user, documentUuid, evaluationUuids: [evaluationUuid], }).then((r) => r.unwrap()) diff --git a/packages/core/src/tests/factories/createProject.ts b/packages/core/src/tests/factories/createProject.ts index 5efebf4a4..aa0baf52b 100644 --- a/packages/core/src/tests/factories/createProject.ts +++ b/packages/core/src/tests/factories/createProject.ts @@ -41,9 +41,12 @@ export async function createProject(projectData: Partial = {}) { }) const { commit: draft } = await createDraft({ project, user }) for await (const { path, content } of documentsToCreate) { - const newDoc = await createNewDocument({ commit: draft, path }).then( - (r) => r.unwrap(), - ) + const newDoc = await createNewDocument({ + commit: draft, + path, + user, + workspace, + }).then((r) => r.unwrap()) const updatedDoc = await updateDocument({ commit: draft, document: newDoc, diff --git a/packages/core/src/tests/factories/documents.ts b/packages/core/src/tests/factories/documents.ts index 8e5036e6d..e5beb08a5 100644 --- a/packages/core/src/tests/factories/documents.ts +++ b/packages/core/src/tests/factories/documents.ts @@ -1,6 +1,6 @@ import { and, eq } from 'drizzle-orm' -import { type Commit } from '../../browser' +import { User, Workspace, type Commit } from '../../browser' import { database } from '../../client' import { documentVersions } from '../../schema' import { createNewDocument } from '../../services/documents/create' @@ -14,7 +14,7 @@ export type IDocumentVersionData = { } export async function markAsSoftDelete( - { commitId, documentUuid }: { commitId: number; documentUuid: string }, + { documentUuid, commitId }: { documentUuid: string; commitId: number }, tx = database, ) { return tx @@ -28,8 +28,12 @@ export async function markAsSoftDelete( ) } -export async function createDocumentVersion(data: IDocumentVersionData) { +export async function createDocumentVersion( + data: IDocumentVersionData & { workspace: Workspace; user: User }, +) { let result = await createNewDocument({ + workspace: data.workspace, + user: data.user, commit: data.commit, path: data.path, }) diff --git a/packages/core/src/tests/factories/evaluations.ts b/packages/core/src/tests/factories/evaluations.ts index 86ea7af14..f90db9e6b 100644 --- a/packages/core/src/tests/factories/evaluations.ts +++ b/packages/core/src/tests/factories/evaluations.ts @@ -4,12 +4,14 @@ import { EvaluationMetadataType, EvaluationResultableType, EvaluationResultConfiguration, + User, Workspace, } from '../../browser' import { createEvaluation as createEvaluationService } from '../../services/evaluations' export type IEvaluationData = { workspace: Workspace + user: User name?: string description?: string prompt?: string @@ -18,6 +20,7 @@ export type IEvaluationData = { export async function createLlmAsJudgeEvaluation({ workspace, + user, name, description, prompt, @@ -27,6 +30,7 @@ export async function createLlmAsJudgeEvaluation({ }: IEvaluationData) { const evaluationResult = await createEvaluationService({ workspace, + user, metadata: { prompt: prompt ?? faker.lorem.sentence() }, type: EvaluationMetadataType.LlmAsJudge, name: name ?? faker.company.catchPhrase(), diff --git a/packages/core/src/tests/factories/projects.ts b/packages/core/src/tests/factories/projects.ts index 4912112ad..da312dfca 100644 --- a/packages/core/src/tests/factories/projects.ts +++ b/packages/core/src/tests/factories/projects.ts @@ -47,7 +47,7 @@ export type ICreateProject = { deletedAt?: Date | null workspace?: Workspace | ICreateWorkspace providers?: { type: Providers; name: string }[] - evaluations?: Omit[] + evaluations?: Omit[] documents?: IDocumentStructure skipMerge?: boolean } @@ -101,7 +101,7 @@ export async function createProject(projectData: Partial = {}) { const evaluations = await Promise.all( projectData.evaluations?.map((evaluationData) => - createLlmAsJudgeEvaluation({ workspace, ...evaluationData }), + createLlmAsJudgeEvaluation({ workspace, user, ...evaluationData }), ) ?? [], ) @@ -113,9 +113,12 @@ export async function createProject(projectData: Partial = {}) { }) const { commit: draft } = await createDraft({ project, user }) for await (const { path, content } of documentsToCreate) { - const newDoc = await createNewDocument({ commit: draft, path }).then( - (r) => r.unwrap(), - ) + const newDoc = await createNewDocument({ + workspace, + user, + commit: draft, + path, + }).then((r) => r.unwrap()) const updatedDoc = await updateDocument({ commit: draft, document: newDoc, diff --git a/packages/core/src/tests/factories/providerApiKeys.ts b/packages/core/src/tests/factories/providerApiKeys.ts index 4f6ac50f0..d481ea2db 100644 --- a/packages/core/src/tests/factories/providerApiKeys.ts +++ b/packages/core/src/tests/factories/providerApiKeys.ts @@ -24,7 +24,7 @@ export async function createProviderApiKey({ provider: type, name, token: `sk-${faker.string.alphanumeric(48)}`, - authorId: user.id, + author: user, }).then((r) => r.unwrap()) return providerApiKey diff --git a/packages/jobs/src/constants.ts b/packages/jobs/src/constants.ts index b85fbba8a..e4e42afd1 100644 --- a/packages/jobs/src/constants.ts +++ b/packages/jobs/src/constants.ts @@ -6,6 +6,7 @@ import { runDocumentJob } from './job-definitions/batchEvaluations/runDocumentJo import { runEvaluationJob } from './job-definitions/batchEvaluations/runEvaluationJob' import { createEventJob } from './job-definitions/events/createEventJob' import { publishEventJob } from './job-definitions/events/publishEventJob' +import { publishToAnalyticsJob } from './job-definitions/events/publishToAnalyticsJob' import { runLiveEvaluationJob } from './job-definitions/liveEvaluations/runLiveEvaluationJob' // TODO: Review if we can remove this declarations @@ -36,7 +37,7 @@ export const QUEUES = { }, [Queues.eventsQueue]: { name: Queues.eventsQueue, - jobs: [publishEventJob, createEventJob], + jobs: [publishEventJob, createEventJob, publishToAnalyticsJob], }, [Queues.eventHandlersQueue]: { name: Queues.eventHandlersQueue, diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.test.ts b/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.test.ts index 0410e7d0d..49f72fc1a 100644 --- a/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.test.ts +++ b/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.test.ts @@ -2,7 +2,6 @@ import { randomUUID } from 'crypto' import { Workspace } from '@latitude-data/core/browser' import { findWorkspaceFromDocument } from '@latitude-data/core/data-access' -import { NotFoundError } from '@latitude-data/core/lib/errors' import { previewDataset } from '@latitude-data/core/services/datasets/preview' import { Job } from 'bullmq' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -20,6 +19,12 @@ const mocks = vi.hoisted(() => ({ enqueueRunDocumentJob: vi.fn(), }, }, + eventsQueue: { + jobs: { + enqueueCreateEventJob: vi.fn(), + enqueuePublishEventJob: vi.fn(), + }, + }, }, })) @@ -66,6 +71,7 @@ describe('runBatchEvaluationJob', () => { beforeEach(() => { vi.clearAllMocks() + // @ts-ignore mockJob = { data: { evaluation: { id: 1 }, @@ -74,9 +80,11 @@ describe('runBatchEvaluationJob', () => { commitUuid: 'commit-uuid-1', projectId: 1, parametersMap: { param1: 0, param2: 1 }, - }, + workspace: { id: 'workspace-1' }, + user: { id: 'user-1', email: 'user-1@example.com' }, + } as unknown as Job, attemptsMade: 0, - } as unknown as Job + } // @ts-ignore mockProgressTracker = { @@ -181,13 +189,6 @@ describe('runBatchEvaluationJob', () => { ) }) - it('should throw NotFoundError if workspace is not found', async () => { - // @ts-ignore - vi.mocked(findWorkspaceFromDocument).mockResolvedValue(null) - - await expect(runBatchEvaluationJob(mockJob)).rejects.toThrow(NotFoundError) - }) - it('should resume from last enqueued job on retry', async () => { mockJob.attemptsMade = 1 // @ts-ignore diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts b/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts index 29ec84498..ad00f5497 100644 --- a/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts +++ b/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts @@ -4,9 +4,10 @@ import { Dataset, DocumentVersion, EvaluationDto, + User, + Workspace, } from '@latitude-data/core/browser' -import { findWorkspaceFromDocument } from '@latitude-data/core/data-access' -import { NotFoundError } from '@latitude-data/core/lib/errors' +import { publisher } from '@latitude-data/core/events/publisher' import { queues } from '@latitude-data/core/queues' import { CommitsRepository } from '@latitude-data/core/repositories' import { previewDataset } from '@latitude-data/core/services/datasets/preview' @@ -17,6 +18,8 @@ import { setupJobs } from '../..' import { ProgressTracker } from '../../utils/progressTracker' type RunBatchEvaluationJobParams = { + workspace: Workspace + user: User evaluation: EvaluationDto dataset: Dataset document: DocumentVersion @@ -32,6 +35,8 @@ export const runBatchEvaluationJob = async ( job: Job, ) => { const { + workspace, + user, evaluation, dataset, document, @@ -43,13 +48,20 @@ export const runBatchEvaluationJob = async ( batchId = randomUUID(), } = job.data const websockets = await WebsocketClient.getSocket() - const workspace = await findWorkspaceFromDocument(document) - if (!workspace) throw new NotFoundError('Workspace not found') - const commit = await new CommitsRepository(workspace.id) .getCommitByUuid({ projectId, uuid: commitUuid }) .then((r) => r.unwrap()) const fileMetadata = dataset.fileMetadata + + publisher.publishLater({ + type: 'batchEvaluationRun', + data: { + evaluationId: evaluation.id, + workspaceId: workspace.id, + userEmail: user.email, + }, + }) + // TODO: use streaming instead of this service in order to avoid loading the // whole dataset in memory const result = await previewDataset({ diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts b/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts index 3cdd5948c..14f5adcac 100644 --- a/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts +++ b/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts @@ -74,6 +74,7 @@ describe('runDocumentJob', () => { commit = setup.commit evaluation = await factories.createLlmAsJudgeEvaluation({ + user: setup.user, workspace, name: 'Test Evaluation', }) diff --git a/packages/jobs/src/job-definitions/events/publishToAnalyticsJob.ts b/packages/jobs/src/job-definitions/events/publishToAnalyticsJob.ts new file mode 100644 index 000000000..54bb68bb2 --- /dev/null +++ b/packages/jobs/src/job-definitions/events/publishToAnalyticsJob.ts @@ -0,0 +1,40 @@ +import { LatitudeEvent } from '@latitude-data/core/events/handlers/index' +import { PostHogClient } from '@latitude-data/core/services/posthog' +import { Job } from 'bullmq' + +export const publishToAnalyticsJob = async (job: Job) => { + const event = job.data + let userEmail, workspaceId + if ('userEmail' in event.data) { + userEmail = event.data.userEmail + } + if ('workspaceId' in event.data) { + workspaceId = event.data.workspaceId + } + + const client = PostHogClient() + + if (!userEmail && !workspaceId) return + + if (!userEmail) { + client.capture({ + distinctId: `workspace:${workspaceId}`, + event: event.type, + properties: { + ...event.data, + workspaceId, + }, + }) + } else { + client.capture({ + distinctId: userEmail, + event: event.type, + properties: { + ...event.data, + workspaceId, + }, + }) + } + + await client.shutdown() +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e40fc4054..bc8a22e60 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -134,6 +134,9 @@ importers: pg: specifier: ^8.12.0 version: 8.13.0 + posthog-node: + specifier: ^4.2.0 + version: 4.2.0 react: specifier: ^18.3.1 version: 18.3.1 @@ -332,7 +335,7 @@ importers: version: 4.6.0(monaco-editor@0.50.0)(react-dom@19.0.0-rc-5d19e1c8-20240923(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923) '@sentry/nextjs': specifier: ^8 - version: 8.32.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@15.0.0-canary.168(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-5d19e1c8-20240923(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923)(webpack@5.94.0(esbuild@0.19.12)) + version: 8.32.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@15.0.0-canary.168(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-5d19e1c8-20240923(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923)(webpack@5.94.0) '@sentry/utils': specifier: ^8.30.0 version: 8.32.0 @@ -717,6 +720,9 @@ importers: pg-transactional-tests: specifier: ^1.0.9 version: 1.1.0(pg@8.13.0) + posthog-node: + specifier: ^4.2.0 + version: 4.2.0 prettier: specifier: ^3.3.3 version: 3.3.3 @@ -2426,6 +2432,9 @@ packages: peerDependencies: drizzle-orm: '>= 0.29 <1' lucia: 3.x + peerDependenciesMeta: + drizzle-orm: + optional: true '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -7991,6 +8000,10 @@ packages: posthog-js@1.165.0: resolution: {integrity: sha512-rUfRJobvOz3Q9Er+zwb32Eq2qs+ToBe/B4k4IoKzmyszI7240Rf4xVWRB0ky8LvmdZfCeYX5knS2Uv3pnn/d5A==} + posthog-node@4.2.0: + resolution: {integrity: sha512-hgyCYMyzMvuF3qWMw6JvS8gT55v7Mtp5wKWcnDrw+nu39D0Tk9BXD7I0LOBp0lGlHEPaXCEVYUtviNKrhMALGA==} + engines: {node: '>=15.0.0'} + preact@10.24.1: resolution: {integrity: sha512-PnBAwFI3Yjxxcxw75n6VId/5TFxNW/81zexzWD9jn1+eSrOP84NdsS38H5IkF/UH3frqRPT+MvuCoVHjTDTnDw==} @@ -8388,6 +8401,9 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + rusha@0.8.14: + resolution: {integrity: sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==} + safe-array-concat@1.1.2: resolution: {integrity: sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==} engines: {node: '>=0.4'} @@ -11036,8 +11052,9 @@ snapshots: '@lucia-auth/adapter-drizzle@1.1.0(drizzle-orm@0.33.0(@opentelemetry/api@1.9.0)(@types/pg@8.11.10)(@types/react@18.3.0)(pg@8.13.0)(react@19.0.0-rc-5d19e1c8-20240923))(lucia@3.2.0)': dependencies: - drizzle-orm: 0.33.0(@opentelemetry/api@1.9.0)(@types/pg@8.11.10)(@types/react@18.3.0)(pg@8.13.0)(react@19.0.0-rc-5d19e1c8-20240923) lucia: 3.2.0 + optionalDependencies: + drizzle-orm: 0.33.0(@opentelemetry/api@1.9.0)(@types/pg@8.11.10)(@types/react@18.3.0)(pg@8.13.0)(react@19.0.0-rc-5d19e1c8-20240923) '@lukeed/ms@2.0.2': {} @@ -13004,7 +13021,7 @@ snapshots: '@sentry/types': 8.32.0 '@sentry/utils': 8.32.0 - '@sentry/nextjs@8.32.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@15.0.0-canary.168(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-5d19e1c8-20240923(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923)(webpack@5.94.0(esbuild@0.19.12))': + '@sentry/nextjs@8.32.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@15.0.0-canary.168(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-5d19e1c8-20240923(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923)(webpack@5.94.0)': dependencies: '@opentelemetry/instrumentation-http': 0.53.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.27.0 @@ -13017,14 +13034,14 @@ snapshots: '@sentry/types': 8.32.0 '@sentry/utils': 8.32.0 '@sentry/vercel-edge': 8.32.0 - '@sentry/webpack-plugin': 2.22.3(encoding@0.1.13)(webpack@5.94.0(esbuild@0.19.12)) + '@sentry/webpack-plugin': 2.22.3(encoding@0.1.13)(webpack@5.94.0) chalk: 3.0.0 next: 15.0.0-canary.168(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-5d19e1c8-20240923(react@19.0.0-rc-5d19e1c8-20240923))(react@19.0.0-rc-5d19e1c8-20240923) resolve: 1.22.8 rollup: 3.29.5 stacktrace-parser: 0.1.10 optionalDependencies: - webpack: 5.94.0(esbuild@0.19.12) + webpack: 5.94.0 transitivePeerDependencies: - '@opentelemetry/api' - '@opentelemetry/core' @@ -13105,12 +13122,12 @@ snapshots: '@sentry/types': 8.32.0 '@sentry/utils': 8.32.0 - '@sentry/webpack-plugin@2.22.3(encoding@0.1.13)(webpack@5.94.0(esbuild@0.19.12))': + '@sentry/webpack-plugin@2.22.3(encoding@0.1.13)(webpack@5.94.0)': dependencies: '@sentry/bundler-plugin-core': 2.22.3(encoding@0.1.13) unplugin: 1.0.1 uuid: 9.0.1 - webpack: 5.94.0(esbuild@0.19.12) + webpack: 5.94.0 transitivePeerDependencies: - encoding - supports-color @@ -13712,7 +13729,7 @@ snapshots: '@types/mute-stream@0.0.4': dependencies: - '@types/node': 22.7.2 + '@types/node': 20.16.9 '@types/mysql@2.15.26': dependencies: @@ -17706,6 +17723,13 @@ snapshots: preact: 10.24.1 web-vitals: 4.2.3 + posthog-node@4.2.0: + dependencies: + axios: 1.7.7 + rusha: 0.8.14 + transitivePeerDependencies: + - debug + preact@10.24.1: {} prelude-ls@1.2.1: {} @@ -18248,6 +18272,8 @@ snapshots: dependencies: queue-microtask: 1.2.3 + rusha@0.8.14: {} + safe-array-concat@1.1.2: dependencies: call-bind: 1.0.7 @@ -18841,16 +18867,14 @@ snapshots: mkdirp: 1.0.4 yallist: 4.0.0 - terser-webpack-plugin@5.3.10(esbuild@0.19.12)(webpack@5.94.0(esbuild@0.19.12)): + terser-webpack-plugin@5.3.10(webpack@5.94.0): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.34.0 - webpack: 5.94.0(esbuild@0.19.12) - optionalDependencies: - esbuild: 0.19.12 + webpack: 5.94.0 terser@5.34.0: dependencies: @@ -19537,7 +19561,7 @@ snapshots: webpack-virtual-modules@0.5.0: {} - webpack@5.94.0(esbuild@0.19.12): + webpack@5.94.0: dependencies: '@types/estree': 1.0.6 '@webassemblyjs/ast': 1.12.1 @@ -19559,7 +19583,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(esbuild@0.19.12)(webpack@5.94.0(esbuild@0.19.12)) + terser-webpack-plugin: 5.3.10(webpack@5.94.0) watchpack: 2.4.2 webpack-sources: 3.2.3 transitivePeerDependencies: