From 3fbadc477ab179029276bca0a5fb48ffc48449b2 Mon Sep 17 00:00:00 2001 From: Gerard Clos Date: Fri, 13 Sep 2024 12:29:24 +0200 Subject: [PATCH] feature: support structured outputs from AI and used them in evaluations Also implemented optionally transactional provider log generation upon calling ai service --- .../drizzle/0052_famous_daimon_hellstrom.sql | 3 + packages/core/drizzle/meta/0052_snapshot.json | 2194 +++++++++++++++++ packages/core/drizzle/meta/_journal.json | 7 + packages/core/package.json | 5 +- packages/core/src/constants.ts | 31 +- .../events/handlers/createProviderLogJob.ts | 2 +- packages/core/src/events/handlers/index.ts | 38 +- .../core/src/schema/models/providerLogs.ts | 5 +- packages/core/src/schema/types.ts | 8 +- packages/core/src/services/ai/index.ts | 164 +- packages/core/src/services/chains/run.test.ts | 292 +++ packages/core/src/services/chains/run.ts | 256 +- packages/core/src/services/evaluations/run.ts | 57 +- .../core/src/services/providerLogs/create.ts | 12 +- .../providerLogs/formatForEvaluation.ts | 7 +- packages/jobs/src/queues/index.ts | 8 +- pnpm-lock.yaml | 34 +- 17 files changed, 2899 insertions(+), 224 deletions(-) create mode 100644 packages/core/drizzle/0052_famous_daimon_hellstrom.sql create mode 100644 packages/core/drizzle/meta/0052_snapshot.json create mode 100644 packages/core/src/services/chains/run.test.ts diff --git a/packages/core/drizzle/0052_famous_daimon_hellstrom.sql b/packages/core/drizzle/0052_famous_daimon_hellstrom.sql new file mode 100644 index 000000000..891e4c380 --- /dev/null +++ b/packages/core/drizzle/0052_famous_daimon_hellstrom.sql @@ -0,0 +1,3 @@ +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "response_text" DROP DEFAULT;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "response_text" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ADD COLUMN "response_object" jsonb; \ No newline at end of file diff --git a/packages/core/drizzle/meta/0052_snapshot.json b/packages/core/drizzle/meta/0052_snapshot.json new file mode 100644 index 000000000..7760ae01f --- /dev/null +++ b/packages/core/drizzle/meta/0052_snapshot.json @@ -0,0 +1,2194 @@ +{ + "id": "9b5a6c7c-2af0-4e63-b502-4f7ae011066d", + "prevId": "eb7ce322-fe62-4bd3-b181-5386c6db6c01", + "version": "7", + "dialect": "postgresql", + "tables": { + "latitude.users": { + "name": "users", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "encrypted_password": { + "name": "encrypted_password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + }, + "latitude.sessions": { + "name": "sessions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.workspaces": { + "name": "workspaces", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_creator_id_users_id_fk": { + "name": "workspaces_creator_id_users_id_fk", + "tableFrom": "workspaces", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.memberships": { + "name": "memberships", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invitation_token": { + "name": "invitation_token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_workspace_id_user_id_index": { + "name": "memberships_workspace_id_user_id_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_invitation_token_index": { + "name": "memberships_invitation_token_index", + "columns": [ + { + "expression": "invitation_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_workspace_id_workspaces_id_fk": { + "name": "memberships_workspace_id_workspaces_id_fk", + "tableFrom": "memberships", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "memberships_invitation_token_unique": { + "name": "memberships_invitation_token_unique", + "nullsNotDistinct": false, + "columns": [ + "invitation_token" + ] + } + } + }, + "latitude.api_keys": { + "name": "api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_id_idx": { + "name": "workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_keys_workspace_id_workspaces_id_fk": { + "name": "api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_token_unique": { + "name": "api_keys_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.projects": { + "name": "projects", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_idx": { + "name": "workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "projects_workspace_id_workspaces_id_fk": { + "name": "projects_workspace_id_workspaces_id_fk", + "tableFrom": "projects", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.commits": { + "name": "commits", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "project_commit_order_idx": { + "name": "project_commit_order_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "commits_project_id_projects_id_fk": { + "name": "commits_project_id_projects_id_fk", + "tableFrom": "commits", + "tableTo": "projects", + "schemaTo": "latitude", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "commits_user_id_users_id_fk": { + "name": "commits_user_id_users_id_fk", + "tableFrom": "commits", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "commits_uuid_unique": { + "name": "commits_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + }, + "unique_commit_version": { + "name": "unique_commit_version", + "nullsNotDistinct": false, + "columns": [ + "version", + "project_id" + ] + } + } + }, + "latitude.document_versions": { + "name": "document_versions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "document_versions_commit_id_commits_id_fk": { + "name": "document_versions_commit_id_commits_id_fk", + "tableFrom": "document_versions", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "unique_document_uuid_commit_id": { + "name": "unique_document_uuid_commit_id", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "commit_id" + ] + }, + "unique_path_commit_id_deleted_at": { + "name": "unique_path_commit_id_deleted_at", + "nullsNotDistinct": false, + "columns": [ + "path", + "commit_id", + "deleted_at" + ] + } + } + }, + "latitude.provider_api_keys": { + "name": "provider_api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "provider_apikeys_workspace_id_idx": { + "name": "provider_apikeys_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_name_idx": { + "name": "provider_apikeys_name_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_user_id_idx": { + "name": "provider_apikeys_user_id_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_api_keys_author_id_users_id_fk": { + "name": "provider_api_keys_author_id_users_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_api_keys_workspace_id_workspaces_id_fk": { + "name": "provider_api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_apikeys_token_provider_unique": { + "name": "provider_apikeys_token_provider_unique", + "nullsNotDistinct": false, + "columns": [ + "token", + "provider", + "workspace_id" + ] + } + } + }, + "latitude.document_logs": { + "name": "document_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parameters": { + "name": "parameters", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "custom_identifier": { + "name": "custom_identifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_log_uuid_idx": { + "name": "document_log_uuid_idx", + "columns": [ + { + "expression": "document_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "commit_id_idx": { + "name": "commit_id_idx", + "columns": [ + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_logs_commit_id_commits_id_fk": { + "name": "document_logs_commit_id_commits_id_fk", + "tableFrom": "document_logs", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "document_logs_uuid_unique": { + "name": "document_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.provider_logs": { + "name": "provider_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_log_uuid": { + "name": "document_log_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "messages": { + "name": "messages", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "response_object": { + "name": "response_object", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "response_text": { + "name": "response_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'[]'::json" + }, + "tokens": { + "name": "tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "cost_in_millicents": { + "name": "cost_in_millicents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "apiKeyId": { + "name": "apiKeyId", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "generated_at": { + "name": "generated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_logs_provider_id_provider_api_keys_id_fk": { + "name": "provider_logs_provider_id_provider_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "provider_api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "provider_logs_apiKeyId_api_keys_id_fk": { + "name": "provider_logs_apiKeyId_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "apiKeyId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_logs_uuid_unique": { + "name": "provider_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.datasets": { + "name": "datasets", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "csv_delimiter": { + "name": "csv_delimiter", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_key": { + "name": "file_key", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "file_metadata": { + "name": "file_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_workspace_idx": { + "name": "datasets_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_author_idx": { + "name": "datasets_author_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_workspace_id_name_index": { + "name": "datasets_workspace_id_name_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_workspace_id_workspaces_id_fk": { + "name": "datasets_workspace_id_workspaces_id_fk", + "tableFrom": "datasets", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_author_id_users_id_fk": { + "name": "datasets_author_id_users_id_fk", + "tableFrom": "datasets", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations": { + "name": "evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata_id": { + "name": "metadata_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "metadata_type", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_workspace_idx": { + "name": "evaluation_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluation_metadata_idx": { + "name": "evaluation_metadata_idx", + "columns": [ + { + "expression": "metadata_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "metadata_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluations_workspace_id_workspaces_id_fk": { + "name": "evaluations_workspace_id_workspaces_id_fk", + "tableFrom": "evaluations", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "evaluations_uuid_unique": { + "name": "evaluations_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.llm_as_judge_evaluation_metadatas": { + "name": "llm_as_judge_evaluation_metadatas", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true, + "default": "'llm_as_judge'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template_id": { + "name": "template_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "llm_as_judge_evaluation_metadatas_template_id_idx": { + "name": "llm_as_judge_evaluation_metadatas_template_id_idx", + "columns": [ + { + "expression": "template_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk": { + "name": "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk", + "tableFrom": "llm_as_judge_evaluation_metadatas", + "tableTo": "evaluations_templates", + "schemaTo": "latitude", + "columnsFrom": [ + "template_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.connected_evaluations": { + "name": "connected_evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "evaluation_mode": { + "name": "evaluation_mode", + "type": "evaluation_mode_enum", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "connected_evaluations_evaluation_idx": { + "name": "connected_evaluations_evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "connected_evaluations_evaluation_id_evaluations_id_fk": { + "name": "connected_evaluations_evaluation_id_evaluations_id_fk", + "tableFrom": "connected_evaluations", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "connected_evaluations_unique_idx": { + "name": "connected_evaluations_unique_idx", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "evaluation_id" + ] + } + } + }, + "latitude.evaluation_results": { + "name": "evaluation_results", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "document_log_id": { + "name": "document_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider_log_id": { + "name": "provider_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resultable_type": { + "name": "resultable_type", + "type": "evaluation_result_types", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "resultable_id": { + "name": "resultable_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_idx": { + "name": "evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_log_idx": { + "name": "document_log_idx", + "columns": [ + { + "expression": "document_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_log_idx": { + "name": "provider_log_idx", + "columns": [ + { + "expression": "provider_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resultable_idx": { + "name": "resultable_idx", + "columns": [ + { + "expression": "resultable_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resultable_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluation_results_evaluation_id_evaluations_id_fk": { + "name": "evaluation_results_evaluation_id_evaluations_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "evaluation_results_document_log_id_document_logs_id_fk": { + "name": "evaluation_results_document_log_id_document_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "document_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "document_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "evaluation_results_provider_log_id_provider_logs_id_fk": { + "name": "evaluation_results_provider_log_id_provider_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "provider_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_templates": { + "name": "evaluations_templates", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "evaluations_templates_category_evaluations_template_categories_id_fk": { + "name": "evaluations_templates_category_evaluations_template_categories_id_fk", + "tableFrom": "evaluations_templates", + "tableTo": "evaluations_template_categories", + "schemaTo": "latitude", + "columnsFrom": [ + "category" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_template_categories": { + "name": "evaluations_template_categories", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.magic_link_tokens": { + "name": "magic_link_tokens", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "expired_at": { + "name": "expired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "magic_link_tokens_user_id_users_id_fk": { + "name": "magic_link_tokens_user_id_users_id_fk", + "tableFrom": "magic_link_tokens", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "magic_link_tokens_token_unique": { + "name": "magic_link_tokens_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.events": { + "name": "events", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "event_type_idx": { + "name": "event_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_numbers": { + "name": "evaluation_resultable_numbers", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_texts": { + "name": "evaluation_resultable_texts", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_booleans": { + "name": "evaluation_resultable_booleans", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "latitude.provider": { + "name": "provider", + "schema": "latitude", + "values": [ + "openai", + "anthropic", + "groq", + "mistral", + "azure", + "google" + ] + }, + "latitude.log_source": { + "name": "log_source", + "schema": "latitude", + "values": [ + "playground", + "api", + "evaluation" + ] + }, + "latitude.metadata_type": { + "name": "metadata_type", + "schema": "latitude", + "values": [ + "llm_as_judge" + ] + }, + "public.evaluation_result_types": { + "name": "evaluation_result_types", + "schema": "public", + "values": [ + "evaluation_resultable_booleans", + "evaluation_resultable_texts", + "evaluation_resultable_numbers" + ] + } + }, + "schemas": { + "latitude": "latitude" + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/core/drizzle/meta/_journal.json b/packages/core/drizzle/meta/_journal.json index 2b594cf46..d25411418 100644 --- a/packages/core/drizzle/meta/_journal.json +++ b/packages/core/drizzle/meta/_journal.json @@ -365,6 +365,13 @@ "when": 1726148717688, "tag": "0051_colorful_lightspeed", "breakpoints": true + }, + { + "idx": 52, + "version": "7", + "when": 1726227047167, + "tag": "0052_famous_daimon_hellstrom", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/core/package.json b/packages/core/package.json index be2b5e3f3..628b36e3e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -47,6 +47,7 @@ "@latitude-data/typescript-config": "workspace:*", "@sindresorhus/slugify": "^2.2.1", "@t3-oss/env-core": "^0.11.1", + "@types/json-schema": "^7.0.15", "@types/lodash-es": "^4.17.12", "@types/node": "^22.5.0", "@types/pg": "^8.11.6", @@ -59,6 +60,7 @@ "eslint": "8", "eslint-plugin-drizzle": "^0.2.3", "flydrive": "^1.1.0", + "json-schema": "^0.4.0", "lodash-es": "^4.17.21", "pg": "^8.12.0", "pg-transactional-tests": "^1.0.9", @@ -85,11 +87,12 @@ "@latitude-data/mailers": "workspace:^", "@sindresorhus/slugify": "^2.2.1", "@t3-oss/env-core": "^0.11.1", - "ai": "^3.2.42", + "ai": "^3.3.3", "argon2": "^0.41.0", "csv-parse": "^5.5.6", "drizzle-orm": "^0.33.0", "flydrive": "^1.1.0", + "json-schema": "^0.4.0", "lodash-es": "^4.17.21", "pg": "^8.12.0", "react": "^18.3.1", diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 911d063f3..74f02520f 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -2,7 +2,12 @@ import type { Message as CompilerMessage, ToolCall, } from '@latitude-data/compiler' -import { CompletionTokenUsage, CoreTool, TextStreamPart } from 'ai' +import { + CompletionTokenUsage, + CoreTool, + ObjectStreamPart, + TextStreamPart, +} from 'ai' import { Config } from './services/ai' @@ -34,14 +39,23 @@ export const HELP_CENTER = { commitVersions: `${LATITUDE_DOCS_URL}/not-found`, } -export type ChainStepCallResponse = { +export type ChainStepTextResponse = { text: string usage: CompletionTokenUsage + toolCalls: ToolCall[] +} +export type ChainStepObjectResponse = { + object: any + usage: CompletionTokenUsage } -export type ChainCallResponse = ChainStepCallResponse & { + +export type ChainTextResponse = ChainStepTextResponse & { documentLogUuid: string - toolCalls: ToolCall[] } +export type ChainObjectResponse = ChainStepObjectResponse & { + documentLogUuid: string +} +export type ChainCallResponse = ChainTextResponse | ChainObjectResponse export enum Providers { OpenAI = 'openai', @@ -69,7 +83,9 @@ export enum ChainEventTypes { StepComplete = 'chain-step-complete', } -type ProviderData = TextStreamPart> +export type ProviderData = + | TextStreamPart> + | ObjectStreamPart> export type ProviderDataType = ProviderData['type'] type LatitudeEventData = @@ -81,12 +97,13 @@ type LatitudeEventData = } | { type: ChainEventTypes.StepComplete - response: ChainStepCallResponse + response: ChainCallResponse } | { type: ChainEventTypes.Complete config: Config - messages: Message[] + messages?: Message[] + object?: any response: ChainCallResponse } | { diff --git a/packages/core/src/events/handlers/createProviderLogJob.ts b/packages/core/src/events/handlers/createProviderLogJob.ts index 37dce0bbc..95e60a41e 100644 --- a/packages/core/src/events/handlers/createProviderLogJob.ts +++ b/packages/core/src/events/handlers/createProviderLogJob.ts @@ -6,5 +6,5 @@ export const createProviderLogJob = async ({ }: { data: AIProviderCallCompleted }) => { - await createProviderLog(event.data) + return await createProviderLog(event.data).then((r) => r.unwrap()) } diff --git a/packages/core/src/events/handlers/index.ts b/packages/core/src/events/handlers/index.ts index ca7d86c70..8bcab9afe 100644 --- a/packages/core/src/events/handlers/index.ts +++ b/packages/core/src/events/handlers/index.ts @@ -1,18 +1,11 @@ -import { ToolCall } from '@latitude-data/compiler' -import { CompletionTokenUsage } from 'ai' - import { ChainCallResponse, - LogSources, MagicLinkToken, Membership, - Message, - Providers, + ProviderLog, User, } from '../../browser' -import { PartialConfig } from '../../services/ai' import { createEvaluationResultJob } from './createEvaluationResultJob' -import { createProviderLogJob } from './createProviderLogJob' import { createDocumentLogJob } from './documentLogs/createJob' import { sendInvitationToUserJob } from './sendInvitationToUser' import { sendMagicLinkJob } from './sendMagicLinkHandler' @@ -31,24 +24,6 @@ export type EventHandler = ({ data: E }) => void -export type AIProviderCallCompleted = LatitudeEventGeneric< - 'aiProviderCallCompleted', - { - uuid: string - source: LogSources - generatedAt: Date - documentLogUuid?: string - providerId: number - providerType: Providers - model: string - config: PartialConfig - messages: Message[] - responseText: string - toolCalls?: ToolCall[] - usage: CompletionTokenUsage - duration: number - } -> export type MagicLinkTokenCreated = LatitudeEventGeneric< 'magicLinkTokenCreated', MagicLinkToken @@ -83,28 +58,33 @@ export type DocumentRunEvent = LatitudeEventGeneric< } > +export type ProviderLogCreatedEvent = LatitudeEventGeneric< + 'providerLogCreated', + ProviderLog +> + export type LatitudeEvent = | MembershipCreatedEvent | UserCreatedEvent | MagicLinkTokenCreated - | AIProviderCallCompleted | EvaluationRunEvent | DocumentRunEvent + | ProviderLogCreatedEvent export interface IEventsHandlers { - aiProviderCallCompleted: EventHandler[] magicLinkTokenCreated: EventHandler[] membershipCreated: EventHandler[] userCreated: EventHandler[] evaluationRun: EventHandler[] documentRun: EventHandler[] + providerLogCreated: EventHandler[] } export const EventHandlers: IEventsHandlers = { magicLinkTokenCreated: [sendMagicLinkJob], membershipCreated: [sendInvitationToUserJob], userCreated: [], - aiProviderCallCompleted: [createProviderLogJob], evaluationRun: [createEvaluationResultJob], documentRun: [createDocumentLogJob], + providerLogCreated: [], } as const diff --git a/packages/core/src/schema/models/providerLogs.ts b/packages/core/src/schema/models/providerLogs.ts index 0118367b1..097348cb8 100644 --- a/packages/core/src/schema/models/providerLogs.ts +++ b/packages/core/src/schema/models/providerLogs.ts @@ -4,11 +4,13 @@ import { bigserial, integer, json, + jsonb, text, timestamp, uuid, varchar, } from 'drizzle-orm/pg-core' +import { JSONSchema7 } from 'json-schema' import { LogSources } from '../../constants' import { PartialConfig } from '../../services/ai' @@ -36,7 +38,8 @@ export const providerLogs = latitudeSchema.table('provider_logs', { model: varchar('model'), config: json('config').$type().notNull(), messages: json('messages').$type().notNull(), - responseText: text('response_text').$type().notNull().default(''), + responseObject: jsonb('response_object').$type(), + responseText: text('response_text').$type(), toolCalls: json('tool_calls').$type().notNull().default([]), tokens: bigint('tokens', { mode: 'number' }).notNull(), costInMillicents: integer('cost_in_millicents').notNull().default(0), diff --git a/packages/core/src/schema/types.ts b/packages/core/src/schema/types.ts index a3009e2e1..25bbffff9 100644 --- a/packages/core/src/schema/types.ts +++ b/packages/core/src/schema/types.ts @@ -1,4 +1,3 @@ -import { ToolCall } from '@latitude-data/compiler' import { type InferSelectModel } from 'drizzle-orm' import { EvaluationResultableType } from '../constants' @@ -38,12 +37,7 @@ export type ApiKey = InferSelectModel export type Commit = InferSelectModel export type DocumentVersion = InferSelectModel export type Project = InferSelectModel -export type ProviderLog = InferSelectModel & { - // Typescript thinks these 2 are optional because they are in the schema - // but we add a default empty string and empty array to them - responseText: string - toolCalls: ToolCall[] -} +export type ProviderLog = InferSelectModel export type DocumentLog = InferSelectModel export type Evaluation = InferSelectModel export type ConnectedEvaluation = InferSelectModel diff --git a/packages/core/src/services/ai/index.ts b/packages/core/src/services/ai/index.ts index ed2627ddb..c1b47373c 100644 --- a/packages/core/src/services/ai/index.ts +++ b/packages/core/src/services/ai/index.ts @@ -4,19 +4,31 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google' import { createMistral } from '@ai-sdk/mistral' import { createOpenAI } from '@ai-sdk/openai' import { Message } from '@latitude-data/compiler' +import { setupJobs } from '@latitude-data/jobs' import { CallWarning, CompletionTokenUsage, CoreMessage, FinishReason, + streamObject, streamText, } from 'ai' +import { JSONSchema7 } from 'json-schema' import { v4 } from 'uuid' import { z } from 'zod' -import { LogSources, ProviderApiKey, Providers } from '../../browser' +import { + LogSources, + ProviderApiKey, + ProviderLog, + Providers, +} from '../../browser' +import { AIProviderCallCompleted } from '../../events/handlers' import { publisher } from '../../events/publisher' -import { CreateProviderLogProps } from '../providerLogs/create' +import { + createProviderLog, + CreateProviderLogProps, +} from '../providerLogs/create' export type FinishCallbackEvent = { finishReason: FinishReason @@ -95,28 +107,29 @@ function createProvider({ } export type AILog = Omit -export async function ai( - { - provider: apiProvider, - prompt, - messages, - config, - documentLogUuid, - source, - }: { - provider: ProviderApiKey - config: PartialConfig - messages: Message[] - documentLogUuid?: string - prompt?: string - source: LogSources - }, - { - onFinish, - }: { - onFinish?: FinishCallback - } = {}, -) { +export async function ai({ + provider: apiProvider, + prompt, + messages, + config, + documentLogUuid, + source, + schema = config.schema, + output = config.output, + transactionalLogs = false, + onFinish, +}: { + provider: ProviderApiKey + config: PartialConfig + messages: Message[] + documentLogUuid?: string + prompt?: string + source: LogSources + schema?: JSONSchema7 + output?: 'object' | 'array' | 'no-schema' + transactionalLogs?: boolean + onFinish?: FinishCallback +}) { const startTime = Date.now() const { provider, @@ -127,43 +140,82 @@ export async function ai( const model = config.model const m = createProvider({ provider, apiKey, config })(model) - const result = await streamText({ + const commonOptions = { model: m, prompt, messages: messages as CoreMessage[], - onFinish: (event) => { - publisher.publish({ - type: 'aiProviderCallCompleted', - data: { - uuid: v4(), - source, - generatedAt: new Date(), - documentLogUuid, - providerId, - providerType, - model, - config, - messages, - responseText: event.text, - toolCalls: event.toolCalls?.map((t) => ({ - id: t.toolCallId, - name: t.toolName, - arguments: t.args, - })), - usage: event.usage, - duration: Date.now() - startTime, - }, - }) + } - onFinish?.(event) - }, - }) + const createFinishHandler = (isStructured: boolean) => async (event: any) => { + const commonData = { + uuid: v4(), + source, + generatedAt: new Date(), + documentLogUuid, + providerId, + providerType, + model, + config, + messages, + toolCalls: event.toolCalls?.map((t: any) => ({ + id: t.toolCallId, + name: t.toolName, + arguments: t.args, + })), + usage: event.usage, + duration: Date.now() - startTime, + } + + const payload = { + type: 'aiProviderCallCompleted', + data: { + ...commonData, + responseText: event.text, + responseObject: isStructured ? event.object : undefined, + }, + } + + let providerLogUuid + if (transactionalLogs) { + const providerLog = await createProviderLog(payload.data).then((r) => + r.unwrap(), + ) + providerLogUuid = providerLog.uuid + } else { + await setupJobs().defaultQueue.jobs.enqueueCreateProviderLogJob( + payload.data, + ) + } + + onFinish?.({ ...event, providerLogUuid }) + } + + if (schema && output) { + const result = await streamObject({ + ...commonOptions, + schema, + // @ts-expect-error - TODO: fix this + output, + onFinish: createFinishHandler(true), + }) + + return { + fullStream: result.fullStream, + object: result.object, + usage: result.usage, + } + } else { + const result = await streamText({ + ...commonOptions, + onFinish: createFinishHandler(false), + }) - return { - fullStream: result.fullStream, - text: result.text, - usage: result.usage, - toolCalls: result.toolCalls, + return { + fullStream: result.fullStream, + text: result.text, + usage: result.usage, + toolCalls: result.toolCalls, + } } } diff --git a/packages/core/src/services/chains/run.test.ts b/packages/core/src/services/chains/run.test.ts new file mode 100644 index 000000000..3a9fcbd77 --- /dev/null +++ b/packages/core/src/services/chains/run.test.ts @@ -0,0 +1,292 @@ +import { Chain, ContentType, MessageRole } from '@latitude-data/compiler' +import { v4 as uuid } from 'uuid' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { LogSources, Providers } from '../../constants' +import * as factories from '../../tests/factories' +import * as aiModule from '../ai' +import { runChain } from './run' + +// Mock other dependencies +vi.mock('@latitude-data/compiler') +vi.mock('uuid') + +describe('runChain', () => { + const mockChain: Partial = { + step: vi.fn(), + rawText: 'Test raw text', + } + + const mockUUID = '12345678-1234-1234-1234-123456789012' + let apikeys: Map + + function createMockAiResponse(text: string, totalTokens: number) { + return { + text: Promise.resolve(text), + usage: Promise.resolve({ totalTokens }), + toolCalls: Promise.resolve([]), + fullStream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: 'text', text }) + controller.close() + }, + }), + } + } + + beforeEach(async () => { + vi.resetAllMocks() + vi.mocked(uuid).mockReturnValue(mockUUID) + + const { providers } = await factories.createProject({ + providers: [{ name: 'openai', type: Providers.OpenAI }], + }) + apikeys = new Map(providers.map((p) => [p.name, p])) + }) + + it('runs a chain without schema override', async () => { + const mockAiResponse = createMockAiResponse('AI response', 10) + vi.spyOn(aiModule, 'ai').mockResolvedValue(mockAiResponse as any) + + vi.mocked(mockChain.step!).mockResolvedValue({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Test message' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + text: 'AI response', + usage: { totalTokens: 10 }, + toolCalls: [], + }) + + expect(aiModule.ai).toHaveBeenCalledWith( + expect.objectContaining({ + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + schema: undefined, + output: undefined, + }), + ) + }) + + it('runs a chain with schema override', async () => { + const mockSchema = { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'number' }, + }, + } as const + + const mockAiResponse = { + object: Promise.resolve({ name: 'John', age: 30 }), + usage: Promise.resolve({ totalTokens: 15 }), + fullStream: new ReadableStream({ + start(controller) { + controller.enqueue({ + type: 'object', + object: { name: 'John', age: 30 }, + }) + controller.close() + }, + }), + } + + vi.spyOn(aiModule, 'ai').mockResolvedValue(mockAiResponse as any) + + vi.mocked(mockChain.step!).mockResolvedValue({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Test message' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + configOverrides: { + schema: mockSchema, + output: 'object', + }, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + object: { name: 'John', age: 30 }, + usage: { totalTokens: 15 }, + }) + + expect(aiModule.ai).toHaveBeenCalledWith( + expect.objectContaining({ + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + schema: mockSchema, + output: 'object', + }), + ) + }) + + it('handles errors during chain execution', async () => { + vi.mocked(mockChain.step!).mockRejectedValue( + new Error('Chain execution failed'), + ) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + await expect(result.value.response).rejects.toThrow( + 'Chain execution failed', + ) + await expect(result.value.duration).rejects.toThrow( + 'Chain execution failed', + ) + }) + + it('handles multiple steps in a chain', async () => { + const mockAiResponse1 = createMockAiResponse('AI response 1', 10) + const mockAiResponse2 = createMockAiResponse('AI response 2', 15) + + vi.spyOn(aiModule, 'ai') + .mockResolvedValueOnce(mockAiResponse1 as any) + .mockResolvedValueOnce(mockAiResponse2 as any) + + vi.mocked(mockChain.step!) + .mockResolvedValueOnce({ + completed: false, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Step 1' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + .mockResolvedValueOnce({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Step 1' }], + }, + { + role: MessageRole.assistant, + content: 'AI response 1', + toolCalls: [], + }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Step 2' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + text: 'AI response 2', + usage: { totalTokens: 15 }, + toolCalls: [], + }) + + expect(aiModule.ai).toHaveBeenCalledTimes(2) + }) + + it('handles system messages correctly', async () => { + const mockAiResponse = createMockAiResponse('AI response', 10) + vi.spyOn(aiModule, 'ai').mockResolvedValue(mockAiResponse as any) + + vi.mocked(mockChain.step!).mockResolvedValue({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.system, + content: 'System instruction', + }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'User message' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + text: 'AI response', + usage: { totalTokens: 10 }, + toolCalls: [], + }) + + expect(aiModule.ai).toHaveBeenCalledWith( + expect.objectContaining({ + messages: [ + { role: MessageRole.system, content: 'System instruction' }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'User message' }], + }, + ], + }), + ) + }) +}) diff --git a/packages/core/src/services/chains/run.ts b/packages/core/src/services/chains/run.ts index d5e8d67da..9fe13de96 100644 --- a/packages/core/src/services/chains/run.ts +++ b/packages/core/src/services/chains/run.ts @@ -1,4 +1,6 @@ import { Chain, MessageRole } from '@latitude-data/compiler' +import { CoreTool, ObjectStreamPart, TextStreamPart } from 'ai' +import { JSONSchema7 } from 'json-schema' import { v4 } from 'uuid' import { ZodError } from 'zod' @@ -7,7 +9,9 @@ import { ChainCallResponse, ChainEvent, ChainEventTypes, + ChainTextResponse, LogSources, + ProviderData, StreamEventTypes, } from '../../constants' import { NotFoundError, Result, UnprocessableEntityError } from '../../lib' @@ -21,11 +25,16 @@ export async function runChain({ apikeys, source, generateUUID = v4, + configOverrides, }: { chain: Chain generateUUID?: () => string source: LogSources apikeys: CachedApiKeys + configOverrides?: { + schema: JSONSchema7 + output: 'object' | 'array' | 'no-schema' + } }) { const documentLogUuid = generateUUID() @@ -46,6 +55,7 @@ export async function runChain({ apikeys, controller, documentLogUuid, + configOverrides, }) .then(responseResolve) .catch(responseReject) @@ -70,6 +80,7 @@ async function iterate({ previousCount = 0, previousResponse, documentLogUuid, + configOverrides, }: { source: LogSources chain: Chain @@ -78,20 +89,14 @@ async function iterate({ previousCount?: number previousApiKey?: ProviderApiKey documentLogUuid: string - previousResponse?: { - text: string - usage: Record + previousResponse?: ChainTextResponse + configOverrides?: { + schema: JSONSchema7 + output: 'object' | 'array' | 'no-schema' } }) { try { - const { - newMessagesInStep, - conversation, - completed, - config, - apiKey, - sentCount, - } = await doChainStep({ + const stepResult = await computeStepData({ chain, previousResponse, apikeys, @@ -99,35 +104,89 @@ async function iterate({ sentCount: previousCount, }) - enqueueChainEvent(controller, { - data: { - type: ChainEventTypes.Step, - isLastStep: completed, - config: conversation.config as Config, - messages: newMessagesInStep, - }, - event: StreamEventTypes.Latitude, - }) + publishStepStartEvent(controller, stepResult) - const result = await ai({ + const aiResult = await ai({ source, documentLogUuid, - messages: conversation.messages, - config: config, - provider: apiKey, + messages: stepResult.conversation.messages, + config: stepResult.config, + provider: stepResult.apiKey, + schema: configOverrides?.schema, + output: configOverrides?.output, }) - for await (const value of streamToGenerator(result.fullStream)) { - enqueueChainEvent(controller, { - event: StreamEventTypes.Provider, - data: value, + await streamAIResult(controller, aiResult) + + const response = await createChainResponse(aiResult, documentLogUuid) + + if (stepResult.completed) { + await handleCompletedChain(controller, stepResult, response) + return response + } else { + publishStepCompleteEvent(controller, response) + + return iterate({ + source, + chain, + documentLogUuid, + apikeys, + controller, + previousApiKey: stepResult.apiKey, + previousCount: stepResult.sentCount, + previousResponse: response as ChainTextResponse, + configOverrides, }) } + } catch (error) { + handleIterationError(controller, error) + throw error + } +} + +// Helper functions - // TODO: The type on `ai` package return a different definition for `toolCalls` - // than `@latitude-data/compiler` package. We have to unify the naming. - // For now no toolCalls - const response: ChainCallResponse = { +function publishStepStartEvent( + controller: ReadableStreamDefaultController, + stepResult: Awaited>, +) { + enqueueChainEvent(controller, { + data: { + type: ChainEventTypes.Step, + isLastStep: stepResult.completed, + config: stepResult.conversation.config as Config, + messages: stepResult.newMessagesInStep, + }, + event: StreamEventTypes.Latitude, + }) +} + +async function streamAIResult( + controller: ReadableStreamDefaultController, + result: Awaited>, +) { + for await (const value of streamToGenerator< + TextStreamPart> | ObjectStreamPart + >(result.fullStream)) { + enqueueChainEvent(controller, { + event: StreamEventTypes.Provider, + data: value as unknown as ProviderData, + }) + } +} + +async function createChainResponse( + result: Awaited>, + documentLogUuid: string, +): Promise { + if (result.object) { + return { + object: await result.object, + usage: await result.usage, + documentLogUuid, + } + } else { + return { documentLogUuid, text: await result.text, usage: await result.usage, @@ -137,80 +196,87 @@ async function iterate({ arguments: t.args, })), } + } +} - if (completed) { - const completedResponse = { - ...response, - documentLogUuid, - } - enqueueChainEvent(controller, { - event: StreamEventTypes.Latitude, - data: { - type: ChainEventTypes.Complete, - config: conversation.config as Config, - messages: [ - { - role: MessageRole.assistant, - toolCalls: response.toolCalls, - content: response.text, - }, - ], - response: completedResponse, +async function handleCompletedChain( + controller: ReadableStreamDefaultController, + stepResult: Awaited>, + response: ChainCallResponse, +) { + const eventData = { + type: ChainEventTypes.Complete, + config: stepResult.conversation.config as Config, + response, + } as const + + if ('text' in response) { + Object.assign(eventData, { + messages: [ + { + role: MessageRole.assistant, + toolCalls: response.toolCalls || [], + content: response.text || '', }, - }) + ], + }) + } else if ('object' in response) { + Object.assign(eventData, { object: response.object }) + } - controller.close() + enqueueChainEvent(controller, { + event: StreamEventTypes.Latitude, + data: eventData, + }) - return completedResponse - } else { - enqueueChainEvent(controller, { - event: StreamEventTypes.Latitude, - data: { - type: ChainEventTypes.StepComplete, - response, - }, - }) - return iterate({ - source, - chain, - documentLogUuid, - apikeys, - controller, - previousApiKey: apiKey, - previousCount: sentCount, - previousResponse: response, - }) - } - } catch (e) { - const error = e as Error - enqueueChainEvent(controller, { - event: StreamEventTypes.Latitude, - data: { - type: ChainEventTypes.Error, - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, + controller.close() +} + +function publishStepCompleteEvent( + controller: ReadableStreamDefaultController, + response: ChainCallResponse, +) { + enqueueChainEvent(controller, { + event: StreamEventTypes.Latitude, + data: { + type: ChainEventTypes.StepComplete, + response: response, + }, + }) +} + +function handleIterationError( + controller: ReadableStreamDefaultController, + error: unknown, +) { + const chainError = + error instanceof Error ? error : new Error('An unknown error occurred') + + enqueueChainEvent(controller, { + event: StreamEventTypes.Latitude, + data: { + type: ChainEventTypes.Error, + error: { + name: chainError.name, + message: chainError.message, + stack: chainError.stack, }, - }) - controller.close() + }, + }) + controller.close() - if (error instanceof ZodError) { - throw new UnprocessableEntityError( - 'Error validating document configuration', - error.formErrors.fieldErrors, - ) - } else { - throw error - } + if (error instanceof ZodError) { + throw new UnprocessableEntityError( + 'Error validating document configuration', + error.formErrors.fieldErrors, + ) } } /** * Performs some common operations needed for processing an iteration step **/ -async function doChainStep({ +async function computeStepData({ chain, apikeys, previousResponse, @@ -219,7 +285,7 @@ async function doChainStep({ }: { chain: Chain apikeys: CachedApiKeys - previousResponse?: { text: string; usage: Record } + previousResponse?: ChainTextResponse apiKey?: ProviderApiKey sentCount: number }) { diff --git a/packages/core/src/services/evaluations/run.ts b/packages/core/src/services/evaluations/run.ts index fb80db647..7c351e258 100644 --- a/packages/core/src/services/evaluations/run.ts +++ b/packages/core/src/services/evaluations/run.ts @@ -1,6 +1,12 @@ import { createChain, readMetadata } from '@latitude-data/compiler' +import { JSONSchema7 } from 'json-schema' -import { DocumentLog, EvaluationDto, LogSources } from '../../browser' +import { + DocumentLog, + EvaluationDto, + EvaluationResultableType, + LogSources, +} from '../../browser' import { database } from '../../client' import { findLastProviderLogFromDocumentLogUuid } from '../../data-access' import { publisher } from '../../events/publisher' @@ -10,6 +16,20 @@ import { computeDocumentLogWithMetadata } from '../documentLogs' import { buildProviderApikeysMap } from '../providerApiKeys/buildMap' import { formatContext, formatConversation } from '../providerLogs' +// Helper function to get the result schema based on evaluation type +const getResultSchema = (type: EvaluationResultableType): JSONSchema7 => { + switch (type) { + case EvaluationResultableType.Boolean: + return { type: 'boolean' } + case EvaluationResultableType.Number: + return { type: 'number' } + case EvaluationResultableType.Text: + return { type: 'string' } + default: + throw new Error(`Unsupported evaluation type: ${type}`) + } +} + export const runEvaluation = async ( { documentLog, @@ -32,12 +52,11 @@ export const runEvaluation = async ( ) } - const rezult = await computeDocumentLogWithMetadata(documentLog) - if (rezult.error) return rezult - const documentLogWithMetadata = rezult.value + const documentLogWithMetadataResult = + await computeDocumentLogWithMetadata(documentLog) + if (documentLogWithMetadataResult.error) return documentLogWithMetadataResult + const documentLogWithMetadata = documentLogWithMetadataResult.value - // TODO: This will need to become polymorphic in the future given different - // types of evaluations const metadata = await readMetadata({ prompt: documentLog.resolvedContent }) const chain = createChain({ prompt: evaluation.metadata.prompt, @@ -50,21 +69,37 @@ export const runEvaluation = async ( config: metadata.config, duration: documentLogWithMetadata.duration, cost: documentLogWithMetadata.costInMillicents - ? documentLogWithMetadata.costInMillicents * 1000 + ? documentLogWithMetadata.costInMillicents / 1000 : 0, }, }) - const result = await runChain({ + // Use the helper function to get the result schema + const resultSchema = getResultSchema(evaluation.configuration.type) + + const schema: JSONSchema7 = { + type: 'object', + properties: { + result: resultSchema, + reason: { type: 'string' }, + }, + required: ['result', 'reason'], + } + + const chainResult = await runChain({ chain, source: LogSources.Evaluation, apikeys: await buildProviderApikeysMap({ workspaceId: evaluation.workspaceId, }), + configOverrides: { + schema, + output: 'object', + }, }) - if (result.error) return result + if (chainResult.error) return chainResult - result.value.response.then((response) => { + chainResult.value.response.then((response) => { publisher.publish({ type: 'evaluationRun', data: { @@ -76,5 +111,5 @@ export const runEvaluation = async ( }) }) - return result + return chainResult } diff --git a/packages/core/src/services/providerLogs/create.ts b/packages/core/src/services/providerLogs/create.ts index 73a4cb057..a6856a236 100644 --- a/packages/core/src/services/providerLogs/create.ts +++ b/packages/core/src/services/providerLogs/create.ts @@ -1,8 +1,10 @@ import { Message, ToolCall } from '@latitude-data/compiler' import { CompletionTokenUsage } from 'ai' +import { JSONSchema7 } from 'json-schema' import { LogSources, ProviderLog, Providers } from '../../browser' import { database } from '../../client' +import { publisher } from '../../events/publisher' import { Result, Transaction } from '../../lib' import { providerLogs } from '../../schema' import { estimateCost, PartialConfig } from '../ai' @@ -19,7 +21,8 @@ export type CreateProviderLogProps = { model: string config: PartialConfig messages: Message[] - responseText: string + responseText?: string + responseObject?: JSONSchema7 toolCalls?: ToolCall[] usage: CompletionTokenUsage duration: number @@ -38,6 +41,7 @@ export async function createProviderLog( config, messages, responseText, + responseObject, toolCalls, usage, duration, @@ -67,6 +71,7 @@ export async function createProviderLog( config, messages, responseText, + responseObject, toolCalls, tokens: isNaN(usage.totalTokens) ? 0 : (usage.totalTokens ?? 0), costInMillicents: cost, @@ -80,6 +85,11 @@ export async function createProviderLog( await touchProviderApiKey(providerId, trx) if (apiKeyId) await touchApiKey(apiKeyId, trx) + publisher.publishLater({ + type: 'providerLogCreated', + data: log, + }) + return Result.ok(log) }, db) } diff --git a/packages/core/src/services/providerLogs/formatForEvaluation.ts b/packages/core/src/services/providerLogs/formatForEvaluation.ts index 7eb76b0a9..adf24a2f3 100644 --- a/packages/core/src/services/providerLogs/formatForEvaluation.ts +++ b/packages/core/src/services/providerLogs/formatForEvaluation.ts @@ -5,13 +5,18 @@ import { Message, ProviderLog } from '../../browser' export function formatConversation(providerLog: ProviderLog) { const messages: Message[] = [...(providerLog.messages || [])] - // Add the response as an assistant message if it exists if (providerLog.responseText) { messages.push({ role: MessageRole.assistant, content: providerLog.responseText, toolCalls: providerLog.toolCalls, }) + } else if (providerLog.responseObject) { + messages.push({ + role: MessageRole.assistant, + content: JSON.stringify(providerLog.responseObject), + toolCalls: [], + }) } return formatMessages(messages) diff --git a/packages/jobs/src/queues/index.ts b/packages/jobs/src/queues/index.ts index ce1f902df..3b9f85c06 100644 --- a/packages/jobs/src/queues/index.ts +++ b/packages/jobs/src/queues/index.ts @@ -1,3 +1,4 @@ +import { createProviderLogJob } from '@latitude-data/core/events/handlers/createProviderLogJob' import { EventHandlers } from '@latitude-data/core/events/handlers/index' import { Job, JobsOptions, Queue, QueueEvents } from 'bullmq' @@ -66,7 +67,12 @@ function setupQueue({ export const QUEUES = { [Queues.defaultQueue]: { name: Queues.defaultQueue, - jobs: [runBatchEvaluationJob, runDocumentJob, runEvaluationJob], + jobs: [ + runBatchEvaluationJob, + runDocumentJob, + runEvaluationJob, + createProviderLogJob, + ], }, [Queues.eventsQueue]: { name: Queues.eventsQueue, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c08c6ff94..ea78fe11b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -311,7 +311,7 @@ importers: version: 4.6.0(monaco-editor@0.50.0)(react-dom@19.0.0-rc-f994737d14-20240522(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522) '@sentry/nextjs': specifier: ^8 - version: 8.30.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@14.3.0-canary.87(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-f994737d14-20240522(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522)(webpack@5.94.0(esbuild@0.19.12)) + version: 8.30.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@14.3.0-canary.87(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-f994737d14-20240522(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522)(webpack@5.94.0) '@t3-oss/env-nextjs': specifier: ^0.10.1 version: 0.10.1(typescript@5.6.2)(zod@3.23.8) @@ -560,6 +560,9 @@ importers: '@t3-oss/env-core': specifier: ^0.11.1 version: 0.11.1(typescript@5.6.2)(zod@3.23.8) + '@types/json-schema': + specifier: ^7.0.15 + version: 7.0.15 '@types/lodash-es': specifier: ^4.17.12 version: 4.17.12 @@ -596,6 +599,9 @@ importers: flydrive: specifier: ^1.1.0 version: 1.1.0(@aws-sdk/client-s3@3.649.0)(@aws-sdk/s3-request-presigner@3.649.0) + json-schema: + specifier: ^0.4.0 + version: 0.4.0 lodash-es: specifier: ^4.17.21 version: 4.17.21 @@ -2286,6 +2292,9 @@ packages: peerDependencies: drizzle-orm: '>= 0.29 <1' lucia: 3.x + peerDependenciesMeta: + drizzle-orm: + optional: true '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} @@ -10475,8 +10484,9 @@ snapshots: '@lucia-auth/adapter-drizzle@1.1.0(drizzle-orm@0.33.0(@opentelemetry/api@1.9.0)(@types/pg@8.11.9)(@types/react@18.3.0)(pg@8.12.0)(react@19.0.0-rc-f994737d14-20240522))(lucia@3.2.0)': dependencies: - drizzle-orm: 0.33.0(@opentelemetry/api@1.9.0)(@types/pg@8.11.9)(@types/react@18.3.0)(pg@8.12.0)(react@19.0.0-rc-f994737d14-20240522) lucia: 3.2.0 + optionalDependencies: + drizzle-orm: 0.33.0(@opentelemetry/api@1.9.0)(@types/pg@8.11.9)(@types/react@18.3.0)(pg@8.12.0)(react@19.0.0-rc-f994737d14-20240522) '@lukeed/ms@2.0.2': {} @@ -12407,7 +12417,7 @@ snapshots: '@sentry/types': 8.30.0 '@sentry/utils': 8.30.0 - '@sentry/nextjs@8.30.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@14.3.0-canary.87(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-f994737d14-20240522(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522)(webpack@5.94.0(esbuild@0.19.12))': + '@sentry/nextjs@8.30.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@14.3.0-canary.87(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-f994737d14-20240522(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522)(webpack@5.94.0)': dependencies: '@opentelemetry/instrumentation-http': 0.53.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.27.0 @@ -12419,14 +12429,14 @@ snapshots: '@sentry/types': 8.30.0 '@sentry/utils': 8.30.0 '@sentry/vercel-edge': 8.30.0 - '@sentry/webpack-plugin': 2.22.3(encoding@0.1.13)(webpack@5.94.0(esbuild@0.19.12)) + '@sentry/webpack-plugin': 2.22.3(encoding@0.1.13)(webpack@5.94.0) chalk: 3.0.0 next: 14.3.0-canary.87(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-f994737d14-20240522(react@19.0.0-rc-f994737d14-20240522))(react@19.0.0-rc-f994737d14-20240522) resolve: 1.22.8 rollup: 3.29.4 stacktrace-parser: 0.1.10 optionalDependencies: - webpack: 5.94.0(esbuild@0.19.12) + webpack: 5.94.0 transitivePeerDependencies: - '@opentelemetry/api' - '@opentelemetry/core' @@ -12505,12 +12515,12 @@ snapshots: '@sentry/types': 8.30.0 '@sentry/utils': 8.30.0 - '@sentry/webpack-plugin@2.22.3(encoding@0.1.13)(webpack@5.94.0(esbuild@0.19.12))': + '@sentry/webpack-plugin@2.22.3(encoding@0.1.13)(webpack@5.94.0)': dependencies: '@sentry/bundler-plugin-core': 2.22.3(encoding@0.1.13) unplugin: 1.0.1 uuid: 9.0.1 - webpack: 5.94.0(esbuild@0.19.12) + webpack: 5.94.0 transitivePeerDependencies: - encoding - supports-color @@ -17714,16 +17724,14 @@ snapshots: mkdirp: 1.0.4 yallist: 4.0.0 - terser-webpack-plugin@5.3.10(esbuild@0.19.12)(webpack@5.94.0(esbuild@0.19.12)): + terser-webpack-plugin@5.3.10(webpack@5.94.0): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.32.0 - webpack: 5.94.0(esbuild@0.19.12) - optionalDependencies: - esbuild: 0.19.12 + webpack: 5.94.0 terser@5.32.0: dependencies: @@ -18378,7 +18386,7 @@ snapshots: webpack-virtual-modules@0.5.0: {} - webpack@5.94.0(esbuild@0.19.12): + webpack@5.94.0: dependencies: '@types/estree': 1.0.5 '@webassemblyjs/ast': 1.12.1 @@ -18400,7 +18408,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(esbuild@0.19.12)(webpack@5.94.0(esbuild@0.19.12)) + terser-webpack-plugin: 5.3.10(webpack@5.94.0) watchpack: 2.4.2 webpack-sources: 3.2.3 transitivePeerDependencies: