diff --git a/cli/src/client/contract.ts b/cli/src/client/contract.ts index 07aed2fd..1adca0ed 100644 --- a/cli/src/client/contract.ts +++ b/cli/src/client/contract.ts @@ -48,7 +48,7 @@ export const VersionedTextsSchema = z.object({ z.object({ version: z.string(), content: z.string(), - }), + }) ), }); @@ -87,17 +87,13 @@ export const resultDataSchema = z export const learningSchema = z.object({ summary: z .string() - .describe( - "The new information that was learned. Be generic, do not refer to the entities.", - ), + .describe("The new information that was learned. Be generic, do not refer to the entities."), entities: z .array( z.object({ - name: z - .string() - .describe("The name of the entity this learning relates to."), + name: z.string().describe("The name of the entity this learning relates to."), type: z.enum(["tool"]), - }), + }) ) .describe("The entities this learning relates to."), relevance: z.object({ @@ -121,7 +117,7 @@ export const agentDataSchema = z toolName: z.string(), reasoning: z.string().optional(), input: z.object({}).passthrough(), - }), + }) ) .optional(), }) @@ -162,7 +158,7 @@ export const definition = { description: z.string().optional(), schema: z.string().optional(), config: FunctionConfigSchema.optional(), - }), + }) ) .optional(), }), @@ -226,13 +222,11 @@ export const definition = { message: z.string(), }), }, - body: blobSchema - .omit({ id: true, createdAt: true, jobId: true, workflowId: true }) - .and( - z.object({ - data: z.string(), - }), - ), + body: blobSchema.omit({ id: true, createdAt: true, jobId: true, workflowId: true }).and( + z.object({ + data: z.string(), + }) + ), }, getContract: { method: "GET", @@ -256,7 +250,7 @@ export const definition = { name: z.string(), createdAt: z.date(), description: z.string().nullable(), - }), + }) ), 401: z.undefined(), }, @@ -271,9 +265,7 @@ export const definition = { 204: z.undefined(), }, body: z.object({ - description: z - .string() - .describe("Human readable description of the cluster"), + description: z.string().describe("Human readable description of the cluster"), }), }, upsertIntegrations: { @@ -332,13 +324,13 @@ export const definition = { name: z.string().optional(), description: z.string().optional(), additionalContext: VersionedTextsSchema.optional().describe( - "Additional cluster context which is included in all runs", + "Additional cluster context which is included in all runs" ), debug: z .boolean() .optional() .describe( - "Enable additional logging (Including prompts and results) for use by Inferable support", + "Enable additional logging (Including prompts and results) for use by Inferable support" ), enableRunConfigs: z.boolean().optional(), enableKnowledgebase: z.boolean().optional(), @@ -389,7 +381,7 @@ export const definition = { workflowId: z.string().nullable(), meta: z.any().nullable(), id: z.string(), - }), + }) ), 401: z.undefined(), 404: z.undefined(), @@ -437,10 +429,7 @@ export const definition = { .string() .optional() .describe("An initial 'human' message to trigger the run"), - systemPrompt: z - .string() - .optional() - .describe("A system prompt for the run."), + systemPrompt: z.string().optional().describe("A system prompt for the run."), name: z .string() .optional() @@ -452,24 +441,20 @@ export const definition = { resultSchema: anyObject .optional() .describe( - "A JSON schema definition which the result object should conform to. By default the result will be a JSON object which does not conform to any schema", + "A JSON schema definition which the result object should conform to. By default the result will be a JSON object which does not conform to any schema" ), attachedFunctions: z .array(functionReference) .optional() .describe( - "An array of functions to make available to the run. By default all functions in the cluster will be available", + "An array of functions to make available to the run. By default all functions in the cluster will be available" ), onStatusChange: z .object({ - function: functionReference.describe( - "A function to call when the run status changes", - ), + function: functionReference.describe("A function to call when the run status changes"), }) .optional() - .describe( - "Mechanism for receiving notifications when the run status changes", - ), + .describe("Mechanism for receiving notifications when the run status changes"), metadata: z .record(z.string()) .optional() @@ -480,21 +465,16 @@ export const definition = { mocks: z .record( z.object({ - output: z - .object({}) - .passthrough() - .describe("The mock output of the function"), - }), + output: z.object({}).passthrough().describe("The mock output of the function"), + }) ) .optional() .describe( - "Function mocks to be used in the run. (Keys should be function in the format _)", + "Function mocks to be used in the run. (Keys should be function in the format _)" ), }) .optional() - .describe( - "When provided, the run will be marked as as a test / evaluation", - ), + .describe("When provided, the run will be marked as as a test / evaluation"), config: z .object({ id: z.string().describe("The run configuration ID"), @@ -502,7 +482,7 @@ export const definition = { .object({}) .passthrough() .describe( - "The run configuration input arguments, the schema must match the run configuration input schema", + "The run configuration input arguments, the schema must match the run configuration input schema" ) .optional(), }) @@ -517,11 +497,7 @@ export const definition = { }) .optional() .describe("DEPRECATED"), - reasoningTraces: z - .boolean() - .default(true) - .optional() - .describe("Enable reasoning traces"), + reasoningTraces: z.boolean().default(true).optional().describe("Enable reasoning traces"), callSummarization: z .boolean() .default(true) @@ -530,9 +506,7 @@ export const definition = { interactive: z .boolean() .default(true) - .describe( - "Allow the run to be continued with follow-up messages / message edits", - ), + .describe("Allow the run to be continued with follow-up messages / message edits"), }), responses: { 201: z.object({ @@ -609,7 +583,7 @@ export const definition = { createdAt: z.date(), pending: z.boolean().default(false), displayableContext: z.record(z.string()).nullable(), - }), + }) ), 401: z.undefined(), }, @@ -624,13 +598,10 @@ export const definition = { userId: z.string().optional(), test: z.coerce .string() - .transform((value) => value === "true") + .transform(value => value === "true") .optional(), limit: z.coerce.number().min(10).max(50).default(50), - metadata: z - .string() - .optional() - .describe("Filter runs by a metadata value (value:key)"), + metadata: z.string().optional().describe("Filter runs by a metadata value (value:key)"), configId: z.string().optional(), }), responses: { @@ -640,14 +611,12 @@ export const definition = { name: z.string(), userId: z.string().nullable(), createdAt: z.date(), - status: z - .enum(["pending", "running", "paused", "done", "failed"]) - .nullable(), + status: z.enum(["pending", "running", "paused", "done", "failed"]).nullable(), test: z.boolean(), configId: z.string().nullable(), configVersion: z.number().nullable(), feedbackScore: z.number().nullable(), - }), + }) ), 401: z.undefined(), }, @@ -662,9 +631,7 @@ export const definition = { 200: z.object({ id: z.string(), userId: z.string().nullable(), - status: z - .enum(["pending", "running", "paused", "done", "failed"]) - .nullable(), + status: z.enum(["pending", "running", "paused", "done", "failed"]).nullable(), failureReason: z.string().nullable(), test: z.boolean(), feedbackComment: z.string().nullable(), @@ -684,12 +651,7 @@ export const definition = { }), body: z.object({ comment: z.string().describe("Feedback comment").nullable(), - score: z - .number() - .describe("Score between 0 and 1") - .min(0) - .max(1) - .nullable(), + score: z.number().describe("Score between 0 and 1").min(0).max(1).nullable(), }), responses: { 204: z.undefined(), @@ -811,7 +773,7 @@ export const definition = { targetFn: z.string(), service: z.string(), executingMachineId: z.string().nullable(), - }), + }) ), }, }, @@ -847,7 +809,7 @@ export const definition = { createdAt: z.date(), createdBy: z.string(), revokedAt: z.date().nullable(), - }), + }) ), }, }, @@ -877,7 +839,7 @@ export const definition = { id: z.string(), lastPingAt: z.date(), ip: z.string(), - }), + }) ), }, pathParams: z.object({ @@ -903,10 +865,10 @@ export const definition = { description: z.string().optional(), schema: z.string().optional(), config: FunctionConfigSchema.optional(), - }), + }) ) .optional(), - }), + }) ), }, pathParams: z.object({ @@ -945,7 +907,7 @@ export const definition = { createdAt: z.date(), pending: z.boolean().default(false), displayableContext: z.record(z.string()).nullable(), - }), + }) ), activity: z.array( z.object({ @@ -956,7 +918,7 @@ export const definition = { createdAt: z.date(), jobId: z.string().nullable(), targetFn: z.string().nullable(), - }), + }) ), jobs: z.array( z.object({ @@ -968,14 +930,12 @@ export const definition = { createdAt: z.date(), approved: z.boolean().nullable(), approvalRequested: z.boolean().nullable(), - }), + }) ), run: z.object({ id: z.string(), userId: z.string().nullable(), - status: z - .enum(["pending", "running", "paused", "done", "failed"]) - .nullable(), + status: z.enum(["pending", "running", "paused", "done", "failed"]).nullable(), failureReason: z.string().nullable(), test: z.boolean(), feedbackComment: z.string().nullable(), @@ -1082,7 +1042,7 @@ export const definition = { resultSchema: anyObject.nullable(), inputSchema: anyObject.nullable(), public: z.boolean(), - }), + }) ), }), 401: z.undefined(), @@ -1103,10 +1063,7 @@ export const definition = { body: z.object({ name: z.string(), initialPrompt: z.string().optional(), - systemPrompt: z - .string() - .optional() - .describe("The initial system prompt for the run."), + systemPrompt: z.string().optional().describe("The initial system prompt for the run."), attachedFunctions: z.array(z.string()).optional(), resultSchema: anyObject.optional(), inputSchema: z.object({}).passthrough().optional().nullable(), @@ -1131,10 +1088,7 @@ export const definition = { body: z.object({ name: z.string().optional(), initialPrompt: z.string().optional(), - systemPrompt: z - .string() - .optional() - .describe("The initial system prompt for the run."), + systemPrompt: z.string().optional().describe("The initial system prompt for the run."), attachedFunctions: z.array(z.string()).optional(), resultSchema: z.object({}).passthrough().optional().nullable(), inputSchema: z.object({}).passthrough().optional().nullable(), @@ -1187,7 +1141,7 @@ export const definition = { resultSchema: z.unknown().nullable(), createdAt: z.date(), updatedAt: z.date(), - }), + }) ), 401: z.undefined(), }, @@ -1214,7 +1168,7 @@ export const definition = { createdAt: z.date(), updatedAt: z.date(), similarity: z.number(), - }), + }) ), 401: z.undefined(), }, @@ -1234,7 +1188,7 @@ export const definition = { jobFailureCount: z.number(), timeToCompletion: z.number(), jobCount: z.number(), - }), + }) ), }, pathParams: z.object({ @@ -1251,11 +1205,9 @@ export const definition = { z.object({ id: z.string(), data: z.string(), - tags: z - .array(z.string()) - .transform((tags) => tags.map((tag) => tag.toLowerCase().trim())), + tags: z.array(z.string()).transform(tags => tags.map(tag => tag.toLowerCase().trim())), title: z.string(), - }), + }) ), }), responses: { @@ -1283,7 +1235,7 @@ export const definition = { tags: z.array(z.string()), title: z.string(), similarity: z.number(), - }), + }) ), 401: z.undefined(), }, @@ -1297,9 +1249,7 @@ export const definition = { headers: z.object({ authorization: z.string() }), body: z.object({ data: z.string(), - tags: z - .array(z.string()) - .transform((tags) => tags.map((tag) => tag.toLowerCase().trim())), + tags: z.array(z.string()).transform(tags => tags.map(tag => tag.toLowerCase().trim())), title: z.string(), }), responses: { @@ -1334,7 +1284,7 @@ export const definition = { data: z.string(), tags: z.array(z.string()), title: z.string(), - }), + }) ), 401: z.undefined(), }, @@ -1363,9 +1313,7 @@ export const definition = { .min(0) .max(20) .default(0) - .describe( - "Time in seconds to keep the request open waiting for a response", - ), + .describe("Time in seconds to keep the request open waiting for a response"), }), headers: z.object({ authorization: z.string(), @@ -1413,14 +1361,9 @@ export const definition = { path: "/clusters/:clusterId/calls", query: z.object({ service: z.string(), - status: z - .enum(["pending", "running", "paused", "done", "failed"]) - .default("pending"), + status: z.enum(["pending", "running", "paused", "done", "failed"]).default("pending"), limit: z.coerce.number().min(1).max(20).default(10), - acknowledge: z.coerce - .boolean() - .default(false) - .describe("Should calls be marked as running"), + acknowledge: z.coerce.boolean().default(false).describe("Should calls be marked as running"), }), pathParams: z.object({ clusterId: z.string(), @@ -1442,7 +1385,7 @@ export const definition = { authContext: z.any().nullable(), runContext: z.any().nullable(), approved: z.boolean(), - }), + }) ), }, }, @@ -1474,7 +1417,7 @@ export const definition = { resultSchema: anyObject .optional() .describe( - "A JSON schema definition which the result object should conform to. By default the result will be a JSON object which does not conform to any schema", + "A JSON schema definition which the result object should conform to. By default the result will be a JSON object which does not conform to any schema" ), modelId: z.enum(["claude-3-5-sonnet", "claude-3-haiku"]), temperature: z diff --git a/control-plane/drizzle/0203_sad_speed.sql b/control-plane/drizzle/0203_sad_speed.sql new file mode 100644 index 00000000..1982b16d --- /dev/null +++ b/control-plane/drizzle/0203_sad_speed.sql @@ -0,0 +1,2 @@ +ALTER TABLE "jobs" ALTER COLUMN "remaining_attempts" SET DEFAULT 1;--> statement-breakpoint +ALTER TABLE "machines" DROP COLUMN IF EXISTS "status"; \ No newline at end of file diff --git a/control-plane/drizzle/meta/0203_snapshot.json b/control-plane/drizzle/meta/0203_snapshot.json new file mode 100644 index 00000000..a62e9886 --- /dev/null +++ b/control-plane/drizzle/meta/0203_snapshot.json @@ -0,0 +1,1645 @@ +{ + "id": "77315311-00ed-4b37-b524-803f51667b65", + "prevId": "01cf3714-f560-44db-97d5-2239553f25e9", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "initial_prompt": { + "name": "initial_prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "system_prompt": { + "name": "system_prompt", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "attached_functions": { + "name": "attached_functions", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'[]'::json" + }, + "structured_output": { + "name": "structured_output", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_cluster_id_clusters_id_fk": { + "name": "agents_cluster_id_clusters_id_fk", + "tableFrom": "agents", + "tableTo": "clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "prompt_templates_pkey": { + "name": "prompt_templates_pkey", + "columns": [ + "cluster_id", + "id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.analytics_snapshots": { + "name": "analytics_snapshots", + "schema": "", + "columns": { + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "analytics_snapshots_pkey": { + "name": "analytics_snapshots_pkey", + "columns": [ + "timestamp" + ] + } + }, + "uniqueConstraints": {} + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_keys_secret_hash_index": { + "name": "api_keys_secret_hash_index", + "columns": [ + { + "expression": "secret_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_keys_cluster_id_clusters_id_fk": { + "name": "api_keys_cluster_id_clusters_id_fk", + "tableFrom": "api_keys", + "tableTo": "clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "api_keys_cluster_id_id_pk": { + "name": "api_keys_cluster_id_id_pk", + "columns": [ + "cluster_id", + "id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.blobs": { + "name": "blobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "job_id": { + "name": "job_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "encoding": { + "name": "encoding", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "blobs_cluster_id_job_id_jobs_cluster_id_id_fk": { + "name": "blobs_cluster_id_job_id_jobs_cluster_id_id_fk", + "tableFrom": "blobs", + "tableTo": "jobs", + "columnsFrom": [ + "cluster_id", + "job_id" + ], + "columnsTo": [ + "cluster_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "blobs_cluster_id_workflow_id_workflows_cluster_id_id_fk": { + "name": "blobs_cluster_id_workflow_id_workflows_cluster_id_id_fk", + "tableFrom": "blobs", + "tableTo": "workflows", + "columnsFrom": [ + "cluster_id", + "workflow_id" + ], + "columnsTo": [ + "cluster_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "blobs_cluster_id_id_pk": { + "name": "blobs_cluster_id_id_pk", + "columns": [ + "cluster_id", + "id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.clusters": { + "name": "clusters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "debug": { + "name": "debug", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_custom_auth": { + "name": "enable_custom_auth", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "handle_custom_auth_function": { + "name": "handle_custom_auth_function", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true, + "default": "'default_handleCustomAuth'" + }, + "enable_knowledgebase": { + "name": "enable_knowledgebase", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "description": { + "name": "description", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "additional_context": { + "name": "additional_context", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp (6) with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp (6) with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "clusters_id_org_index": { + "name": "clusters_id_org_index", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.embeddings": { + "name": "embeddings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp (6) with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "embedding_1024": { + "name": "embedding_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "raw_data": { + "name": "raw_data", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "raw_data_hash": { + "name": "raw_data_hash", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "tags": { + "name": "tags", + "type": "json", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "embedding1024Index": { + "name": "embedding1024Index", + "columns": [ + { + "expression": "embedding_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "embeddingsLookupIndex": { + "name": "embeddingsLookupIndex", + "columns": [ + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "raw_data_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "embeddings_cluster_id_id_type_pk": { + "name": "embeddings_cluster_id_id_type_pk", + "columns": [ + "cluster_id", + "id", + "type" + ] + } + }, + "uniqueConstraints": {} + }, + "public.events": { + "name": "events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "machine_id": { + "name": "machine_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "service": { + "name": "service", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "target_fn": { + "name": "target_fn", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "result_type": { + "name": "result_type", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "tool_name": { + "name": "tool_name", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "model_id": { + "name": "model_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "token_usage_input": { + "name": "token_usage_input", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "token_usage_output": { + "name": "token_usage_output", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "attention_level": { + "name": "attention_level", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "meta": { + "name": "meta", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'::json" + } + }, + "indexes": { + "timeline_index": { + "name": "timeline_index", + "columns": [ + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attention_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.integrations": { + "name": "integrations", + "schema": "", + "columns": { + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "toolhouse": { + "name": "toolhouse", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "langfuse": { + "name": "langfuse", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tavily": { + "name": "tavily", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "valtown": { + "name": "valtown", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "slack": { + "name": "slack", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "integrations_cluster_id_clusters_id_fk": { + "name": "integrations_cluster_id_clusters_id_fk", + "tableFrom": "integrations", + "tableTo": "clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "integrations_pkey": { + "name": "integrations_pkey", + "columns": [ + "cluster_id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_fn": { + "name": "target_fn", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "target_args": { + "name": "target_args", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cache_key": { + "name": "cache_key", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_type": { + "name": "result_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "executing_machine_id": { + "name": "executing_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "remaining_attempts": { + "name": "remaining_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resulted_at": { + "name": "resulted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_retrieved_at": { + "name": "last_retrieved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "function_execution_time_ms": { + "name": "function_execution_time_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "timeout_interval_seconds": { + "name": "timeout_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300 + }, + "service": { + "name": "service", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "auth_context": { + "name": "auth_context", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "run_context": { + "name": "run_context", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "approval_requested": { + "name": "approval_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "approved": { + "name": "approved", + "type": "boolean", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "clusterServiceStatusIndex": { + "name": "clusterServiceStatusIndex", + "columns": [ + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "service", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "clusterServiceStatusFnIndex": { + "name": "clusterServiceStatusFnIndex", + "columns": [ + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "service", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_fn", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "jobs_cluster_id_id": { + "name": "jobs_cluster_id_id", + "columns": [ + "cluster_id", + "id" + ] + } + }, + "uniqueConstraints": { + "jobs_id_unique": { + "name": "jobs_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + } + } + }, + "public.machines": { + "name": "machines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "last_ping_at": { + "name": "last_ping_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sdk_version": { + "name": "sdk_version", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "sdk_language": { + "name": "sdk_language", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "ip": { + "name": "ip", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "machines_id_cluster_id": { + "name": "machines_id_cluster_id", + "columns": [ + "id", + "cluster_id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.services": { + "name": "services", + "schema": "", + "columns": { + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "service": { + "name": "service", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "queue_url": { + "name": "queue_url", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "definition": { + "name": "definition", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "services_cluster_id_clusters_id_fk": { + "name": "services_cluster_id_clusters_id_fk", + "tableFrom": "services", + "tableTo": "clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "services_cluster_id_service": { + "name": "services_cluster_id_service", + "columns": [ + "cluster_id", + "service" + ] + } + }, + "uniqueConstraints": {} + }, + "public.tool_metadata": { + "name": "tool_metadata", + "schema": "", + "columns": { + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "service": { + "name": "service", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "user_defined_context": { + "name": "user_defined_context", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_keys": { + "name": "result_keys", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'::json" + } + }, + "indexes": {}, + "foreignKeys": { + "tool_metadata_cluster_id_clusters_id_fk": { + "name": "tool_metadata_cluster_id_clusters_id_fk", + "tableFrom": "tool_metadata", + "tableTo": "clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tool_metadata_pkey": { + "name": "tool_metadata_pkey", + "columns": [ + "cluster_id", + "service", + "function_name" + ] + } + }, + "uniqueConstraints": {} + }, + "public.versioned_entities": { + "name": "versioned_entities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "entity": { + "name": "entity", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "versioned_entities_pkey": { + "name": "versioned_entities_pkey", + "columns": [ + "cluster_id", + "id", + "type", + "version" + ] + } + }, + "uniqueConstraints": {} + }, + "public.workflow_messages": { + "name": "workflow_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp (6) with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp (6) with time zone", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_messages_workflow_id_cluster_id_workflows_id_cluster_id_fk": { + "name": "workflow_messages_workflow_id_cluster_id_workflows_id_cluster_id_fk", + "tableFrom": "workflow_messages", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id", + "cluster_id" + ], + "columnsTo": [ + "id", + "cluster_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workflow_messages_cluster_id_workflow_id_id": { + "name": "workflow_messages_cluster_id_workflow_id_id", + "columns": [ + "cluster_id", + "workflow_id", + "id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.workflow_metadata": { + "name": "workflow_metadata", + "schema": "", + "columns": { + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "workflowMetadataIndex": { + "name": "workflowMetadataIndex", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "value", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_metadata_workflow_id_cluster_id_workflows_id_cluster_id_fk": { + "name": "workflow_metadata_workflow_id_cluster_id_workflows_id_cluster_id_fk", + "tableFrom": "workflow_metadata", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id", + "cluster_id" + ], + "columnsTo": [ + "id", + "cluster_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workflow_metadata_cluster_id_workflow_id_key": { + "name": "workflow_metadata_cluster_id_workflow_id_key", + "columns": [ + "cluster_id", + "workflow_id", + "key" + ] + } + }, + "uniqueConstraints": {} + }, + "public.workflows": { + "name": "workflows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "result_function": { + "name": "result_function", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "result_schema": { + "name": "result_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "system_prompt": { + "name": "system_prompt", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "model_identifier": { + "name": "model_identifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp (6) with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'pending'" + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "debug": { + "name": "debug", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "attached_functions": { + "name": "attached_functions", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'[]'::json" + }, + "test": { + "name": "test", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "test_mocks": { + "name": "test_mocks", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'::json" + }, + "feedback_comment": { + "name": "feedback_comment", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "feedback_score": { + "name": "feedback_score", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "agent_id": { + "name": "agent_id", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "agent_version": { + "name": "agent_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "reasoning_traces": { + "name": "reasoning_traces", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "interactive": { + "name": "interactive", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_summarization": { + "name": "enable_summarization", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_result_grounding": { + "name": "enable_result_grounding", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auth_context": { + "name": "auth_context", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "json", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "workflows_cluster_id_clusters_id_fk": { + "name": "workflows_cluster_id_clusters_id_fk", + "tableFrom": "workflows", + "tableTo": "clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "workflows_cluster_id_id": { + "name": "workflows_cluster_id_id", + "columns": [ + "cluster_id", + "id" + ] + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/drizzle/meta/_journal.json b/control-plane/drizzle/meta/_journal.json index f6fca206..4d10da79 100644 --- a/control-plane/drizzle/meta/_journal.json +++ b/control-plane/drizzle/meta/_journal.json @@ -1415,6 +1415,13 @@ "when": 1735958735361, "tag": "0202_daffy_baron_zemo", "breakpoints": true + }, + { + "idx": 203, + "version": "7", + "when": 1736022381809, + "tag": "0203_sad_speed", + "breakpoints": true } ] } \ No newline at end of file diff --git a/control-plane/src/modules/auth/custom.test.ts b/control-plane/src/modules/auth/custom.test.ts index 9fe1f8e5..699ee131 100644 --- a/control-plane/src/modules/auth/custom.test.ts +++ b/control-plane/src/modules/auth/custom.test.ts @@ -1,6 +1,6 @@ import { AuthenticationError } from "../../utilities/errors"; import { pollJobs } from "../jobs/jobs"; -import { acknowledgeJob, persistJobResult } from "../jobs/persist-result"; +import { acknowledgeJob, persistJobResult } from "../jobs/job-results"; import { editClusterDetails } from "../management"; import { packer } from "../packer"; import { upsertServiceDefinition } from "../service-definitions"; diff --git a/control-plane/src/modules/data.ts b/control-plane/src/modules/data.ts index f8d5fcd8..0b3f5442 100644 --- a/control-plane/src/modules/data.ts +++ b/control-plane/src/modules/data.ts @@ -50,6 +50,14 @@ pool.on("remove", () => { logger.debug("Database connection removed"); }); +// by default jobs have a: +// - timeoutIntervalSeconds: 30 +// - maxAttempts: 1 +export const jobDefaults = { + timeoutIntervalSeconds: 30, + maxAttempts: 1, +}; + export const jobs = pgTable( "jobs", { @@ -69,13 +77,15 @@ export const jobs = pgTable( enum: ["resolution", "rejection", "interrupt"], }), executing_machine_id: text("executing_machine_id"), - remaining_attempts: integer("remaining_attempts").notNull(), + remaining_attempts: integer("remaining_attempts").notNull().default(jobDefaults.maxAttempts), created_at: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), updated_at: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), resulted_at: timestamp("resulted_at", { withTimezone: true }), last_retrieved_at: timestamp("last_retrieved_at", { withTimezone: true }), function_execution_time_ms: integer("function_execution_time_ms"), - timeout_interval_seconds: integer("timeout_interval_seconds").notNull().default(300), + timeout_interval_seconds: integer("timeout_interval_seconds") + .notNull() + .default(jobDefaults.timeoutIntervalSeconds), service: varchar("service", { length: 1024 }).notNull(), workflow_id: varchar("workflow_id", { length: 1024 }).notNull(), auth_context: json("auth_context"), @@ -111,9 +121,6 @@ export const machines = pgTable( sdk_language: varchar("sdk_language", { length: 128 }), ip: varchar("ip", { length: 1024 }).notNull(), cluster_id: varchar("cluster_id").notNull(), - status: text("status", { - enum: ["active", "inactive"], - }).default("active"), }, table => ({ pk: primaryKey({ diff --git a/control-plane/src/modules/jobs/create-job.ts b/control-plane/src/modules/jobs/create-job.ts index 4080c062..d53c875f 100644 --- a/control-plane/src/modules/jobs/create-job.ts +++ b/control-plane/src/modules/jobs/create-job.ts @@ -10,6 +10,7 @@ import { injectTraceContext } from "../observability/tracer"; import { logger } from "../observability/logger"; import { sqs } from "../sqs"; import { externalServices } from "../integrations/constants"; +import { jobDefaults } from "../data"; type CreateJobParams = { jobId: string; @@ -25,8 +26,6 @@ type CreateJobParams = { runContext?: unknown; }; -const DEFAULT_RETRY_COUNT_ON_STALL = 0; - const extractCacheKeyFromJsonPath = (path: string, args: unknown) => { try { return extractWithJsonPath(path, args)[0]; @@ -82,8 +81,10 @@ export const createJob = async (params: { }); const jobConfig = { - timeoutIntervalSeconds: config?.timeoutSeconds, - maxAttempts: (config?.retryCountOnStall ?? DEFAULT_RETRY_COUNT_ON_STALL) + 1, + timeoutIntervalSeconds: config?.timeoutSeconds ?? jobDefaults.timeoutIntervalSeconds, + maxAttempts: config?.retryCountOnStall + ? config?.retryCountOnStall + 1 + : jobDefaults.maxAttempts, jobId: params.toolCallId ?? ulid(), }; diff --git a/control-plane/src/modules/jobs/job-results.ts b/control-plane/src/modules/jobs/job-results.ts new file mode 100644 index 00000000..4a6d91b5 --- /dev/null +++ b/control-plane/src/modules/jobs/job-results.ts @@ -0,0 +1,138 @@ +import { and, eq, isNull, sql } from "drizzle-orm"; +import * as data from "../data"; +import * as events from "../observability/events"; +import { logger } from "../observability/logger"; +import { resumeRun } from "../workflows/workflows"; + +type PersistResultParams = { + result: string; + resultType: "resolution" | "rejection" | "interrupt"; + functionExecutionTime?: number; + jobId: string; + owner: { clusterId: string }; + machineId: string; +}; + +export async function acknowledgeJob({ + jobId, + clusterId, + machineId, +}: { + jobId: string; + clusterId: string; + machineId: string; +}) { + const [job] = await data.db + .update(data.jobs) + .set({ + status: "running", + last_retrieved_at: sql`now()`, + executing_machine_id: machineId, + remaining_attempts: sql`remaining_attempts - 1`, + }) + .where( + and( + eq(data.jobs.id, jobId), + eq(data.jobs.cluster_id, clusterId), + eq(data.jobs.status, "pending") + ) + ) + .returning({ + service: data.jobs.service, + targetFn: data.jobs.target_fn, + targetArgs: data.jobs.target_args, + }); + + if (!job) { + throw new Error(`Failed to acknowledge job ${jobId}`); + } + + events.write({ + type: "jobAcknowledged", + jobId, + clusterId: clusterId, + service: job.service, + machineId, + targetFn: job.targetFn, + meta: { + targetArgs: job.targetArgs, + }, + }); + + return job; +} + +export async function persistJobResult({ + result, + resultType, + functionExecutionTime, + jobId, + owner, + machineId, +}: PersistResultParams) { + const updateResult = await data.db + .update(data.jobs) + .set({ + result, + result_type: resultType, + resulted_at: sql`now()`, + function_execution_time_ms: functionExecutionTime || null, + status: "success", + }) + .where( + and( + eq(data.jobs.id, jobId), + eq(data.jobs.cluster_id, owner.clusterId), + eq(data.jobs.executing_machine_id, machineId), + eq(data.jobs.status, "running"), + isNull(data.jobs.resulted_at) + ) + ) + .returning({ + service: data.jobs.service, + targetFn: data.jobs.target_fn, + runId: data.jobs.workflow_id, + }); + + if (updateResult.length === 0) { + logger.warn("Job result was not persisted", { + jobId, + }); + events.write({ + type: "functionResultedButNotPersisted", + service: updateResult[0]?.service, + clusterId: owner.clusterId, + jobId, + machineId, + targetFn: updateResult[0]?.targetFn, + resultType, + workflowId: updateResult[0]?.runId ?? undefined, + meta: { + functionExecutionTime, + }, + }); + } else { + if (updateResult[0].runId) { + await resumeRun({ + id: updateResult[0].runId, + clusterId: owner.clusterId, + }); + } + + events.write({ + type: "functionResulted", + service: updateResult[0]?.service, + clusterId: owner.clusterId, + jobId, + machineId, + targetFn: updateResult[0]?.targetFn, + resultType, + workflowId: updateResult[0]?.runId ?? undefined, + meta: { + functionExecutionTime, + }, + }); + } + + return updateResult.length; +} diff --git a/control-plane/src/modules/jobs/jobs.test.ts b/control-plane/src/modules/jobs/jobs.test.ts index 7787dea0..94356a5a 100644 --- a/control-plane/src/modules/jobs/jobs.test.ts +++ b/control-plane/src/modules/jobs/jobs.test.ts @@ -3,7 +3,8 @@ import { packer } from "../packer"; import { upsertServiceDefinition } from "../service-definitions"; import { createOwner } from "../test/util"; import { createJob, pollJobs, getJob, requestApproval, submitApproval } from "./jobs"; -import { acknowledgeJob, persistJobResult, selfHealJobs } from "./persist-result"; +import { acknowledgeJob, persistJobResult } from "./job-results"; +import { selfHealCalls } from "./self-heal-jobs"; import * as redis from "../redis"; import { getClusterBackgroundRun } from "../workflows/workflows"; @@ -50,7 +51,7 @@ describe("createJob", () => { }); }); -describe("selfHealJobs", () => { +describe("selfHealCalls", () => { beforeAll(async () => { await redis.start(); }); @@ -104,7 +105,7 @@ describe("selfHealJobs", () => { await new Promise(resolve => setTimeout(resolve, 2000)); // run the self heal job - const healedJobs = await selfHealJobs(); + const healedJobs = await selfHealCalls(); expect(healedJobs.stalledFailedByTimeout).toContain(createJobResult.id); expect(healedJobs.stalledRecovered).toContain(createJobResult.id); @@ -158,7 +159,7 @@ describe("selfHealJobs", () => { await new Promise(resolve => setTimeout(resolve, 2000)); // run the self heal job - const healedJobs = await selfHealJobs(); + const healedJobs = await selfHealCalls(); expect(healedJobs.stalledFailedByTimeout).not.toContain(createJobResult.id); expect(healedJobs.stalledRecovered).not.toContain(createJobResult.id); @@ -214,7 +215,7 @@ describe("selfHealJobs", () => { await new Promise(resolve => setTimeout(resolve, 2000)); // run the self heal job - const healedJobs = await selfHealJobs(); + const healedJobs = await selfHealCalls(); expect(healedJobs.stalledFailedByTimeout).toContain(createJobResult.id); expect(healedJobs.stalledRecovered).not.toContain(createJobResult.id); diff --git a/control-plane/src/modules/jobs/jobs.ts b/control-plane/src/modules/jobs/jobs.ts index e5750e9b..ae03bf2f 100644 --- a/control-plane/src/modules/jobs/jobs.ts +++ b/control-plane/src/modules/jobs/jobs.ts @@ -7,11 +7,11 @@ import * as data from "../data"; import * as events from "../observability/events"; import { packer } from "../packer"; import { resumeRun } from "../workflows/workflows"; -import { selfHealJobs as selfHealCalls } from "./persist-result"; import { notifyApprovalRequest } from "../workflows/notify"; +import { selfHealCalls } from "./self-heal-jobs"; export { createJob } from "./create-job"; -export { acknowledgeJob, persistJobResult } from "./persist-result"; +export { acknowledgeJob, persistJobResult } from "./job-results"; export type ResultType = "resolution" | "rejection" | "interrupt"; diff --git a/control-plane/src/modules/jobs/persist-result.test.ts b/control-plane/src/modules/jobs/persist-result.test.ts index 3f5881ee..fd70b05e 100644 --- a/control-plane/src/modules/jobs/persist-result.test.ts +++ b/control-plane/src/modules/jobs/persist-result.test.ts @@ -2,10 +2,10 @@ import { upsertMachine } from "../machines"; import { upsertServiceDefinition } from "../service-definitions"; import { createOwner } from "../test/util"; import { createJob, getJobStatusSync, persistJobResult } from "./jobs"; -import { acknowledgeJob, selfHealJobs } from "./persist-result"; +import { acknowledgeJob } from "./job-results"; import * as redis from "../redis"; import { getClusterBackgroundRun } from "../workflows/workflows"; - +import { selfHealCalls } from "./self-heal-jobs"; jest.mock("../service-definitions", () => ({ ...jest.requireActual("../service-definitions"), parseJobArgs: jest.fn(), @@ -65,67 +65,6 @@ describe("persistJobResult", () => { }); }); - it("should auto retry when a machine is stalled", async () => { - const owner = await createOwner(); - const targetFn = "machineStallTestFn"; - const targetArgs = "testTargetArgs"; - const service = "testService"; - - await upsertServiceDefinition({ - service: "testService", - definition: { - name: service, - functions: [ - { - name: targetFn, - config: { - retryCountOnStall: 1, - }, - }, - ], - }, - owner, - }); - - const createJobResult = await createJob({ - targetFn, - targetArgs, - owner, - service, - runId: getClusterBackgroundRun(owner.clusterId), - }); - - await upsertMachine({ - clusterId: owner.clusterId, - machineId: "testMachineId", - sdkVersion: "1.0.0", - sdkLanguage: "typescript", - ip: "1.1.1.1", - }); - - // last ping will be now - await acknowledgeJob({ - jobId: createJobResult.id, - clusterId: owner.clusterId, - machineId: "testMachineId", - }); - - const machineStallTimeout = 1; - - // wait 1s for the machine to stall - await new Promise(resolve => setTimeout(resolve, machineStallTimeout * 1000)); - - // self heal jobs with machine stall timeout of 1s - const healedJobs = await selfHealJobs({ machineStallTimeout }); - - expect( - healedJobs.stalledMachines.some( - x => x.id === "testMachineId" && x.clusterId === owner.clusterId - ) - ).toBe(true); - expect(healedJobs.stalledRecovered).toContain(createJobResult.id); - }); - it("should only accept the machine that's assigned to the job", async () => { const owner = await createOwner(); const targetFn = "machineStallTestFn"; diff --git a/control-plane/src/modules/jobs/persist-result.ts b/control-plane/src/modules/jobs/persist-result.ts deleted file mode 100644 index 6035150b..00000000 --- a/control-plane/src/modules/jobs/persist-result.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { and, eq, gt, isNotNull, isNull, lt, lte, or, sql } from "drizzle-orm"; -import * as data from "../data"; -import * as events from "../observability/events"; -import { logger } from "../observability/logger"; -import { resumeRun } from "../workflows/workflows"; - -type PersistResultParams = { - result: string; - resultType: "resolution" | "rejection" | "interrupt"; - functionExecutionTime?: number; - jobId: string; - owner: { clusterId: string }; - machineId: string; -}; - -export async function acknowledgeJob({ - jobId, - clusterId, - machineId, -}: { - jobId: string; - clusterId: string; - machineId: string; -}) { - const [job] = await data.db - .update(data.jobs) - .set({ - status: "running", - last_retrieved_at: sql`now()`, - executing_machine_id: machineId, - remaining_attempts: sql`remaining_attempts - 1`, - }) - .where( - and( - eq(data.jobs.id, jobId), - eq(data.jobs.cluster_id, clusterId), - eq(data.jobs.status, "pending") - ) - ) - .returning({ - service: data.jobs.service, - targetFn: data.jobs.target_fn, - targetArgs: data.jobs.target_args, - }); - - if (!job) { - throw new Error(`Failed to acknowledge job ${jobId}`); - } - - events.write({ - type: "jobAcknowledged", - jobId, - clusterId: clusterId, - service: job.service, - machineId, - targetFn: job.targetFn, - meta: { - targetArgs: job.targetArgs, - }, - }); - - return job; -} - -export async function persistJobResult({ - result, - resultType, - functionExecutionTime, - jobId, - owner, - machineId, -}: PersistResultParams) { - const updateResult = await data.db - .update(data.jobs) - .set({ - result, - result_type: resultType, - resulted_at: sql`now()`, - function_execution_time_ms: functionExecutionTime || null, - status: "success", - }) - .where( - and( - eq(data.jobs.id, jobId), - eq(data.jobs.cluster_id, owner.clusterId), - eq(data.jobs.executing_machine_id, machineId), - eq(data.jobs.status, "running"), - isNull(data.jobs.resulted_at) - ) - ) - .returning({ - service: data.jobs.service, - targetFn: data.jobs.target_fn, - runId: data.jobs.workflow_id, - }); - - if (updateResult.length === 0) { - logger.warn("Job result was not persisted", { - jobId, - }); - events.write({ - type: "functionResultedButNotPersisted", - service: updateResult[0]?.service, - clusterId: owner.clusterId, - jobId, - machineId, - targetFn: updateResult[0]?.targetFn, - resultType, - workflowId: updateResult[0]?.runId ?? undefined, - meta: { - functionExecutionTime, - }, - }); - } else { - if (updateResult[0].runId) { - await resumeRun({ - id: updateResult[0].runId, - clusterId: owner.clusterId, - }); - } - - events.write({ - type: "functionResulted", - service: updateResult[0]?.service, - clusterId: owner.clusterId, - jobId, - machineId, - targetFn: updateResult[0]?.targetFn, - resultType, - workflowId: updateResult[0]?.runId ?? undefined, - meta: { - functionExecutionTime, - }, - }); - } - - return updateResult.length; -} - -export async function selfHealJobs(params?: { machineStallTimeout?: number }) { - // TODO: these queries need to be chunked. If there are 100k jobs, we don't want to update them all at once - - logger.debug("Running Self-healing job"); - - // Jobs are failed if they are running and have timed out - const stalledByTimeout = await data.db - .update(data.jobs) - .set({ - status: "stalled", - }) - .where( - and( - eq(data.jobs.status, "running"), - lt( - data.jobs.last_retrieved_at, - sql`now() - interval '1 second' * timeout_interval_seconds` - ), - // only timeout jobs that have a timeout set - isNotNull(data.jobs.timeout_interval_seconds), - // Don't time out jobs that have pending approval requests - or(eq(data.jobs.approval_requested, false), isNotNull(data.jobs.approved)) - ) - ) - .returning({ - id: data.jobs.id, - service: data.jobs.service, - targetFn: data.jobs.target_fn, - clusterId: data.jobs.cluster_id, - remainingAttempts: data.jobs.remaining_attempts, - runId: data.jobs.workflow_id, - }); - - const stalledMachines = await data.db - .update(data.machines) - .set({ - status: "inactive", - }) - .where( - and( - lt( - data.machines.last_ping_at, - sql`now() - interval '1 second' * ${params?.machineStallTimeout ?? 90}` - ), - eq(data.machines.status, "active") - ) - ) - .returning({ - id: data.machines.id, - clusterId: data.machines.cluster_id, - }); - - // mark jobs with stalled machines as failed - const stalledByMachine = await data.db.execute<{ - id: string; - service: string; - target_fn: string; - cluster_id: string; - remaining_attempts: number; - runId: string | undefined; - }>( - sql` - UPDATE jobs as j - SET status = 'stalled' - FROM machines as m - WHERE - j.status = 'running' AND - j.executing_machine_id = m.id AND - m.status = 'inactive' AND - j.cluster_id = m.cluster_id AND - j.remaining_attempts > 0 - ` - ); - - const stalledRecoveredJobs = await data.db - .update(data.jobs) - .set({ - status: "pending", - remaining_attempts: sql`remaining_attempts - 1`, - }) - .where(and(eq(data.jobs.status, "stalled"), gt(data.jobs.remaining_attempts, 0))) - .returning({ - id: data.jobs.id, - service: data.jobs.service, - targetFn: data.jobs.target_fn, - targetArgs: data.jobs.target_args, - clusterId: data.jobs.cluster_id, - remainingAttempts: data.jobs.remaining_attempts, - }); - - const stalledFailedJobs = await data.db - .update(data.jobs) - .set({ - status: "failure", - }) - .where(and(eq(data.jobs.status, "stalled"), lte(data.jobs.remaining_attempts, 0))) - .returning({ - id: data.jobs.id, - service: data.jobs.service, - targetFn: data.jobs.target_fn, - clusterId: data.jobs.cluster_id, - remainingAttempts: data.jobs.remaining_attempts, - runId: data.jobs.workflow_id, - }); - - stalledByTimeout.forEach(row => { - events.write({ - service: row.service, - clusterId: row.clusterId, - jobId: row.id, - type: "jobStalled", - workflowId: row.runId ?? undefined, - meta: { - attemptsRemaining: row.remainingAttempts ?? undefined, - reason: "timeout", - }, - }); - }); - - stalledByMachine.rows.forEach(row => { - events.write({ - service: row.service, - clusterId: row.cluster_id, - jobId: row.id, - type: "jobStalled", - workflowId: row.runId ?? undefined, - meta: { - attemptsRemaining: row.remaining_attempts ?? undefined, - reason: "machine stalled", - }, - }); - }); - - stalledMachines.forEach(row => { - events.write({ - type: "machineStalled", - clusterId: row.clusterId, - machineId: row.id, - }); - }); - - stalledRecoveredJobs.forEach(async row => { - events.write({ - service: row.service, - clusterId: row.clusterId, - jobId: row.id, - type: "jobRecovered", - }); - }); - - stalledFailedJobs.forEach(row => { - events.write({ - service: row.service, - clusterId: row.clusterId, - jobId: row.id, - type: "jobStalledTooManyTimes", - }); - - if (row.runId) { - resumeRun({ id: row.runId, clusterId: row.clusterId }); - } - }); - - return { - stalledFailedByTimeout: stalledByTimeout.map(row => row.id), - stalledRecovered: stalledRecoveredJobs.map(row => row.id), - stalledMachines: stalledMachines.map(row => ({ - id: row.id, - clusterId: row.clusterId, - })), - stalledFailedByMachine: stalledByMachine.rows.map(row => row.id), - }; -} diff --git a/control-plane/src/modules/jobs/self-heal-jobs.ts b/control-plane/src/modules/jobs/self-heal-jobs.ts new file mode 100644 index 00000000..d4f53d1d --- /dev/null +++ b/control-plane/src/modules/jobs/self-heal-jobs.ts @@ -0,0 +1,102 @@ +import { and, eq, isNotNull, lt, or, sql } from "drizzle-orm"; +import * as data from "../data"; +import * as events from "../observability/events"; +import { logger } from "../observability/logger"; +import { resumeRun } from "../workflows/workflows"; + +export async function selfHealCalls() { + logger.debug("Running Self-healing job"); + + // Jobs are failed if they are running and have timed out + const stalledByTimeout = await data.db + .update(data.jobs) + .set({ + status: "stalled", + }) + .where( + and( + eq(data.jobs.status, "running"), + lt( + data.jobs.last_retrieved_at, + sql`now() - interval '1 second' * timeout_interval_seconds` + ), + // only timeout jobs that have a timeout set + isNotNull(data.jobs.timeout_interval_seconds), + // Don't time out jobs that have pending approval requests + or(eq(data.jobs.approval_requested, false), isNotNull(data.jobs.approved)) + ) + ) + .returning({ + id: data.jobs.id, + service: data.jobs.service, + targetFn: data.jobs.target_fn, + clusterId: data.jobs.cluster_id, + remainingAttempts: data.jobs.remaining_attempts, + runId: data.jobs.workflow_id, + }); + + stalledByTimeout.forEach(row => { + events.write({ + service: row.service, + clusterId: row.clusterId, + jobId: row.id, + type: "jobStalled", + workflowId: row.runId ?? undefined, + meta: { + attemptsRemaining: row.remainingAttempts ?? undefined, + reason: "timeout", + }, + }); + }); + + const stalledJobs = await data.db + .update(data.jobs) + .set({ + status: sql`CASE + WHEN remaining_attempts > 0 THEN 'pending' + ELSE 'failure' + END`, + remaining_attempts: sql`CASE + WHEN remaining_attempts > 0 THEN remaining_attempts - 1 + ELSE remaining_attempts + END`, + }) + .where(eq(data.jobs.status, "stalled")) + .returning({ + id: data.jobs.id, + service: data.jobs.service, + targetFn: data.jobs.target_fn, + targetArgs: data.jobs.target_args, + clusterId: data.jobs.cluster_id, + remainingAttempts: data.jobs.remaining_attempts, + status: data.jobs.status, + runId: data.jobs.workflow_id, + }); + + stalledJobs.forEach(row => { + if (row.status === "pending") { + events.write({ + service: row.service, + clusterId: row.clusterId, + jobId: row.id, + type: "jobRecovered", + }); + } else { + events.write({ + service: row.service, + clusterId: row.clusterId, + jobId: row.id, + type: "jobStalledTooManyTimes", + }); + + if (row.runId) { + resumeRun({ id: row.runId, clusterId: row.clusterId }); + } + } + }); + + return { + stalledFailedByTimeout: stalledByTimeout.map(row => row.id), + stalledRecovered: stalledJobs.filter(row => row.status === "pending").map(row => row.id), + }; +} diff --git a/control-plane/src/modules/machines.test.ts b/control-plane/src/modules/machines.test.ts index 9a0d6f1e..8770a3dc 100644 --- a/control-plane/src/modules/machines.test.ts +++ b/control-plane/src/modules/machines.test.ts @@ -1,10 +1,7 @@ import { upsertMachine } from "./machines"; import { createCluster, getClusterMachines } from "./management"; import * as redis from "./redis"; -import { - getServiceDefinitions, - upsertServiceDefinition, -} from "./service-definitions"; +import { getServiceDefinitions, upsertServiceDefinition } from "./service-definitions"; describe("machines", () => { beforeAll(async () => { @@ -14,7 +11,7 @@ describe("machines", () => { redis.stop(); }); describe("upsertMachine", () => { - it("should record machine details", async () => { + it("should record machine metadata", async () => { const { id } = await createCluster({ organizationId: Math.random().toString(), description: "description", @@ -25,7 +22,7 @@ describe("machines", () => { const services = ["service1", "service2"]; await Promise.all( - services.map((service) => + services.map(service => upsertServiceDefinition({ service, definition: { @@ -33,8 +30,8 @@ describe("machines", () => { functions: [], }, owner: { clusterId: id }, - }), - ), + }) + ) ); const timeBeforePing = Date.now(); @@ -54,14 +51,12 @@ describe("machines", () => { expect(machines).toHaveLength(1); expect(machines[0].id).toBe(machineId); expect(machines[0].lastPingAt).toBeInstanceOf(Date); - expect(machines[0].lastPingAt?.getTime()).toBeGreaterThanOrEqual( - timeBeforePing, - ); + expect(machines[0].lastPingAt?.getTime()).toBeGreaterThanOrEqual(timeBeforePing); expect( await getServiceDefinitions({ clusterId: id, - }).then((services) => services.map((s) => s.service)), + }).then(services => services.map(s => s.service)) ).toEqual(services); }); }); diff --git a/control-plane/src/modules/machines.ts b/control-plane/src/modules/machines.ts index b00372dd..71fca212 100644 --- a/control-plane/src/modules/machines.ts +++ b/control-plane/src/modules/machines.ts @@ -19,7 +19,7 @@ export async function upsertMachine({ ip?: string; }) { if (xForwardedFor && xForwardedFor.split(",").length > 0) { - const hops = xForwardedFor.split(",").map((ip) => ip.trim()); + const hops = xForwardedFor.split(",").map(ip => ip.trim()); if (hops.length > 0 && hops[0]) { ip = hops[0]; @@ -53,19 +53,14 @@ export async function upsertMachine({ sdk_version: sdkVersion, sdk_language: sdkLanguage, cluster_id: clusterId, - status: "active", }) .onConflictDoUpdate({ target: [data.machines.id, data.machines.cluster_id], set: { last_ping_at: sql`excluded.last_ping_at`, ip: sql`excluded.ip`, - status: sql`excluded.status`, }, - where: and( - eq(data.machines.cluster_id, clusterId), - eq(data.machines.id, machineId), - ), - }), + where: and(eq(data.machines.cluster_id, clusterId), eq(data.machines.id, machineId)), + }) ); } diff --git a/control-plane/src/modules/management.ts b/control-plane/src/modules/management.ts index fcf3cf7e..432c9544 100644 --- a/control-plane/src/modules/management.ts +++ b/control-plane/src/modules/management.ts @@ -185,7 +185,8 @@ export const getClusterMachines = async ({ clusterId }: { clusterId: string }) = id: data.machines.id, lastPingAt: data.machines.last_ping_at, ip: data.machines.ip, - status: data.machines.status, + sdkVersion: data.machines.sdk_version, + sdkLanguage: data.machines.sdk_language, }) .from(data.machines) .where(eq(data.machines.cluster_id, clusterId));