From 89686255abc1c63cdcfed005dda3fec859f012aa Mon Sep 17 00:00:00 2001
From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Date: Wed, 13 Nov 2024 07:16:24 +1100
Subject: [PATCH] [8.x] [Obs AI Assistant] Include an AdHoc instruction about
 the slack connector to avoid executing a loop (#199531) (#199860)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Obs AI Assistant] Include an AdHoc instruction about the slack
connector to avoid executing a loop
(#199531)](https://github.com/elastic/kibana/pull/199531)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Viduni
Wickramarachchi","email":"viduni.wickramarachchi@elastic.co"},"sourceCommit":{"committedDate":"2024-11-12T18:28:08Z","message":"[Obs
AI Assistant] Include an AdHoc instruction about the slack connector to
avoid executing a loop (#199531)\n\nCloses
https://github.com/elastic/kibana/issues/185028\r\n\r\n##
Summary\r\n\r\n### Problem\r\nThe ticket mentions that there was an
issue with displaying results.\r\nHowever both `display results` and
`visualize query` are working as\r\nexpected based on my investigation.
More details including a video
are\r\nattached\r\n[here](https://github.com/elastic/kibana/issues/185028#issuecomment-2464879452).\r\n\r\nThe
function calling loop seems to occur when the AI Assistant is
trying\r\nto send the output to Slack via the Kibana Slack connector. In
order to\r\ndo this, the LLM invokes the function `execute_connector`.
For the Slack\r\nconnector, `id` and `params` properties are required.
However, the LLM\r\nonly populated `id` and not `params` which causes an
error when\r\nvalidated against the schema for the Slack
connector.\r\n\r\n- Sometimes, it's able to retry a few times and
successfully send the\r\noutput to Slack.\r\n- Sometimes, it goes into a
loop trying to find `params` and failing\r\nrepeatedly.\r\n\r\nAttaching
another similar issue for more context
-\r\nhttps://github.com/elastic/kibana/issues/195564\r\n\r\n_With the
solution below, my intention is to send the output to Slack in\r\none
go, without retrying the function `execute_connector`._\r\n\r\n###
Solution\r\nBased on the solutions I experimented with, seems like we
need to force\r\nthe LLM to understand what's needed for the Slack
connector. I tried a\r\nfew options here and the combination of updates
that worked are as\r\nfollows:\r\n- Appending an AdHoc instruction about
the Slack connector properties.\r\n(This gives the LLM some additional
information about the required\r\nproperties)\r\n- Updating the
`properties` attached to the connector (when passing the\r\nconnector
list to the LLM), to reflect both `id` and `params`
with\r\n`message`.\r\n\r\nWith the above change, the AI Assistant has
managed to consistently send\r\nthe output to Slack _**without any
retries**_ because of missing\r\n`params`.\r\n\r\n###
Screenshots:\r\n(all alert triggers successfully sent the output to
Slack without having\r\nto retry the function)\r\n\r\n<img
width=\"1452\"
alt=\"success-attempts-to-slack-connector\"\r\nsrc=\"https://github.com/user-attachments/assets/715a5957-2c04-4a55-868f-34abe564f6d4\">","sha":"ed4c0df9ee304ea9bc71ed30c8e5dc2dd9295a00","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","v9.0.0","Team:Obs
AI
Assistant","ci:project-deploy-observability","backport:version","v8.17.0"],"title":"[Obs
AI Assistant] Include an AdHoc instruction about the slack connector to
avoid executing a
loop","number":199531,"url":"https://github.com/elastic/kibana/pull/199531","mergeCommit":{"message":"[Obs
AI Assistant] Include an AdHoc instruction about the slack connector to
avoid executing a loop (#199531)\n\nCloses
https://github.com/elastic/kibana/issues/185028\r\n\r\n##
Summary\r\n\r\n### Problem\r\nThe ticket mentions that there was an
issue with displaying results.\r\nHowever both `display results` and
`visualize query` are working as\r\nexpected based on my investigation.
More details including a video
are\r\nattached\r\n[here](https://github.com/elastic/kibana/issues/185028#issuecomment-2464879452).\r\n\r\nThe
function calling loop seems to occur when the AI Assistant is
trying\r\nto send the output to Slack via the Kibana Slack connector. In
order to\r\ndo this, the LLM invokes the function `execute_connector`.
For the Slack\r\nconnector, `id` and `params` properties are required.
However, the LLM\r\nonly populated `id` and not `params` which causes an
error when\r\nvalidated against the schema for the Slack
connector.\r\n\r\n- Sometimes, it's able to retry a few times and
successfully send the\r\noutput to Slack.\r\n- Sometimes, it goes into a
loop trying to find `params` and failing\r\nrepeatedly.\r\n\r\nAttaching
another similar issue for more context
-\r\nhttps://github.com/elastic/kibana/issues/195564\r\n\r\n_With the
solution below, my intention is to send the output to Slack in\r\none
go, without retrying the function `execute_connector`._\r\n\r\n###
Solution\r\nBased on the solutions I experimented with, seems like we
need to force\r\nthe LLM to understand what's needed for the Slack
connector. I tried a\r\nfew options here and the combination of updates
that worked are as\r\nfollows:\r\n- Appending an AdHoc instruction about
the Slack connector properties.\r\n(This gives the LLM some additional
information about the required\r\nproperties)\r\n- Updating the
`properties` attached to the connector (when passing the\r\nconnector
list to the LLM), to reflect both `id` and `params`
with\r\n`message`.\r\n\r\nWith the above change, the AI Assistant has
managed to consistently send\r\nthe output to Slack _**without any
retries**_ because of missing\r\n`params`.\r\n\r\n###
Screenshots:\r\n(all alert triggers successfully sent the output to
Slack without having\r\nto retry the function)\r\n\r\n<img
width=\"1452\"
alt=\"success-attempts-to-slack-connector\"\r\nsrc=\"https://github.com/user-attachments/assets/715a5957-2c04-4a55-868f-34abe564f6d4\">","sha":"ed4c0df9ee304ea9bc71ed30c8e5dc2dd9295a00"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/199531","number":199531,"mergeCommit":{"message":"[Obs
AI Assistant] Include an AdHoc instruction about the slack connector to
avoid executing a loop (#199531)\n\nCloses
https://github.com/elastic/kibana/issues/185028\r\n\r\n##
Summary\r\n\r\n### Problem\r\nThe ticket mentions that there was an
issue with displaying results.\r\nHowever both `display results` and
`visualize query` are working as\r\nexpected based on my investigation.
More details including a video
are\r\nattached\r\n[here](https://github.com/elastic/kibana/issues/185028#issuecomment-2464879452).\r\n\r\nThe
function calling loop seems to occur when the AI Assistant is
trying\r\nto send the output to Slack via the Kibana Slack connector. In
order to\r\ndo this, the LLM invokes the function `execute_connector`.
For the Slack\r\nconnector, `id` and `params` properties are required.
However, the LLM\r\nonly populated `id` and not `params` which causes an
error when\r\nvalidated against the schema for the Slack
connector.\r\n\r\n- Sometimes, it's able to retry a few times and
successfully send the\r\noutput to Slack.\r\n- Sometimes, it goes into a
loop trying to find `params` and failing\r\nrepeatedly.\r\n\r\nAttaching
another similar issue for more context
-\r\nhttps://github.com/elastic/kibana/issues/195564\r\n\r\n_With the
solution below, my intention is to send the output to Slack in\r\none
go, without retrying the function `execute_connector`._\r\n\r\n###
Solution\r\nBased on the solutions I experimented with, seems like we
need to force\r\nthe LLM to understand what's needed for the Slack
connector. I tried a\r\nfew options here and the combination of updates
that worked are as\r\nfollows:\r\n- Appending an AdHoc instruction about
the Slack connector properties.\r\n(This gives the LLM some additional
information about the required\r\nproperties)\r\n- Updating the
`properties` attached to the connector (when passing the\r\nconnector
list to the LLM), to reflect both `id` and `params`
with\r\n`message`.\r\n\r\nWith the above change, the AI Assistant has
managed to consistently send\r\nthe output to Slack _**without any
retries**_ because of missing\r\n`params`.\r\n\r\n###
Screenshots:\r\n(all alert triggers successfully sent the output to
Slack without having\r\nto retry the function)\r\n\r\n<img
width=\"1452\"
alt=\"success-attempts-to-slack-connector\"\r\nsrc=\"https://github.com/user-attachments/assets/715a5957-2c04-4a55-868f-34abe564f6d4\">","sha":"ed4c0df9ee304ea9bc71ed30c8e5dc2dd9295a00"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Viduni Wickramarachchi <viduni.wickramarachchi@elastic.co>
---
 .../server/functions/execute_connector.ts     |  4 ++-
 .../server/rule_connector/index.ts            | 33 +++++++++++++++++--
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts
index 0088e35a6f6af..70f4ead5e6bb6 100644
--- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts
+++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts
@@ -8,13 +8,15 @@
 import { FunctionRegistrationParameters } from '.';
 import { FunctionVisibility } from '../../common';
 
+export const EXECUTE_CONNECTOR_FUNCTION_NAME = 'execute_connector';
+
 export function registerExecuteConnectorFunction({
   functions,
   resources,
 }: FunctionRegistrationParameters) {
   functions.registerFunction(
     {
-      name: 'execute_connector',
+      name: EXECUTE_CONNECTOR_FUNCTION_NAME,
       description: 'Use this function when user explicitly asks to call a kibana connector.',
       visibility: FunctionVisibility.AssistantOnly,
       parameters: {
diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts
index 59b883fef9c18..33f3bdd2c98f8 100644
--- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts
+++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts
@@ -23,7 +23,6 @@ import {
   JiraParamsSchema,
   PagerdutyParamsSchema,
   SlackApiParamsSchema,
-  SlackParamsSchema,
   WebhookParamsSchema,
 } from '@kbn/stack-connectors-plugin/server';
 import { ObservabilityAIAssistantRouteHandlerResources } from '@kbn/observability-ai-assistant-plugin/server/routes/types';
@@ -37,14 +36,26 @@ import { CompatibleJSONSchema } from '@kbn/observability-ai-assistant-plugin/com
 import { AlertDetailsContextualInsightsService } from '@kbn/observability-plugin/server/services';
 import { getSystemMessageFromInstructions } from '@kbn/observability-ai-assistant-plugin/server/service/util/get_system_message_from_instructions';
 import { AdHocInstruction } from '@kbn/observability-ai-assistant-plugin/common/types';
+import { EXECUTE_CONNECTOR_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/execute_connector';
 import { convertSchemaToOpenApi } from './convert_schema_to_open_api';
 import { OBSERVABILITY_AI_ASSISTANT_CONNECTOR_ID } from '../../common/rule_connector';
 
 const CONNECTOR_PRIVILEGES = ['api:observabilityAIAssistant', 'app:observabilityAIAssistant'];
 
 const connectorParamsSchemas: Record<string, CompatibleJSONSchema> = {
+  '.slack': {
+    type: 'object',
+    properties: {
+      id: { type: 'string' },
+      params: {
+        type: 'object',
+        properties: {
+          message: { type: 'string' },
+        },
+      },
+    },
+  },
   '.slack_api': convertSchemaToOpenApi(SlackApiParamsSchema),
-  '.slack': convertSchemaToOpenApi(SlackParamsSchema),
   '.email': convertSchemaToOpenApi(EmailParamsSchema),
   '.webhook': convertSchemaToOpenApi(WebhookParamsSchema),
   '.jira': convertSchemaToOpenApi(JiraParamsSchema),
@@ -189,6 +200,24 @@ If available, include the link of the conversation at the end of your answer.`
     ),
   };
 
+  const hasSlackConnector = !!connectorsList.filter(
+    (connector) => connector.actionTypeId === '.slack'
+  ).length;
+
+  if (hasSlackConnector && functionClient.hasFunction(EXECUTE_CONNECTOR_FUNCTION_NAME)) {
+    const slackConnectorInstruction: AdHocInstruction = {
+      instruction_type: 'application_instruction',
+      text: dedent(
+        `The execute_connector function can be used to invoke Kibana connectors.
+        To send to the Slack connector, you need the following arguments:
+          - the "id" of the connector
+          - the "params" parameter that you will fill with the message
+        Please include both "id" and "params.message" in the function arguments when executing the Slack connector..`
+      ),
+    };
+    functionClient.registerAdhocInstruction(slackConnectorInstruction);
+  }
+
   const alertsContext = await getAlertsContext(
     execOptions.params.rule,
     execOptions.params.alerts,