[8.x] [inference] improve simulated function calling instructions (#1…

…93414) (#193484)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[inference] improve simulated function calling instructions
(#193414)](#193414)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Pierre
Gayvallet","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-09-19T20:22:44Z","message":"[inference]
improve simulated function calling instructions (#193414)\n\n##
Summary\r\n\r\n(and do it also for
o11y)","sha":"a9402939f378bfd9ba6136b9bba687430c64ca32","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-major","Team:Obs
AI Assistant","ci:project-deploy-observability","v8.16.0","Team:AI
Infra"],"title":"[inference] improve simulated function calling
instructions","number":193414,"url":"https://github.com/elastic/kibana/pull/193414","mergeCommit":{"message":"[inference]
improve simulated function calling instructions (#193414)\n\n##
Summary\r\n\r\n(and do it also for
o11y)","sha":"a9402939f378bfd9ba6136b9bba687430c64ca32"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193414","number":193414,"mergeCommit":{"message":"[inference]
improve simulated function calling instructions (#193414)\n\n##
Summary\r\n\r\n(and do it also for
o11y)","sha":"a9402939f378bfd9ba6136b9bba687430c64ca32"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Pierre Gayvallet <[email protected]>

x-pack/plugins/inference/scripts/util/kibana_client.ts

@@ -179,13 +179,15 @@ export class KibanaClient {
       system,
       toolChoice,
       tools,
+      functionCalling,
     }) => {
       const body: ChatCompleteRequestBody = {
         connectorId: chatCompleteConnectorId,
         system,
         messages,
         toolChoice,
         tools,
+        functionCalling,
       };
 
       return stream(

x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts

@@ -29,9 +29,18 @@ export function getSystemMessageInstructions({
 
     It is EXTREMELY important that you generate valid JSON between the \`\`\`json and \`\`\` delimiters.
 
-    You may call them like this.
+    IMPORTANT: make sure you start and end a tool call with the ${TOOL_USE_START} and ${TOOL_USE_END} markers, it MUST
+    be included in the tool call.
 
-    Given the following tool:
+    You may call tools like this:
+
+    ${TOOL_USE_START}
+    \`\`\`json
+    ${JSON.stringify({ name: '[name of the tool]', input: { myProperty: 'myValue' } })}
+    \`\`\`\
+    ${TOOL_USE_END}
+
+    For example, given the following tool:
 
     ${JSON.stringify({
       name: 'my_tool',
@@ -54,13 +63,15 @@ export function getSystemMessageInstructions({
     \`\`\`\
     ${TOOL_USE_END}
 
-    Given the following tool:
+    Another example: given the following tool:
+
     ${JSON.stringify({
       name: 'my_tool_without_parameters',
       description: 'A tool to call without parameters',
     })}
 
     Use it the following way:
+
     ${TOOL_USE_START}
     \`\`\`json
     ${JSON.stringify({ name: 'my_tool_without_parameters', input: {} })}

x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts

@@ -45,7 +45,7 @@ export function parseInlineFunctionCalls({ logger }: { logger: Logger }) {
           logger.debug('Parsing function call:\n' + buffer);
 
           const match = buffer.match(
-            /<\|tool_use_start\|>\s*```json\n?(.*?)(\n```\s*).*<\|tool_use_end\|>/s
+            /<\|tool_use_start\|>\s*```json\n?(.*?)(\n?```\s*).*<\|tool_use_end\|>/s
           );
 
           const functionCallBody = match?.[1];

x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt

@@ -4,7 +4,7 @@ The BUCKET function allows you to create groups of values, known as buckets, fro
 
 ## Syntax
 
-`BUCKET(field, buckets, from, to)`
+`BUCKET(field, buckets [, from, to])`
 
 ### Parameters
 
@@ -18,15 +18,24 @@ The target number of buckets, or the desired bucket size if `from` and `to` para
 
 #### from
 
-The start of the range. This can be a number, a date, or a date expressed as a string.
+(optional) The start of the range. This can be a number, a date, or a date expressed as a string.
 
 #### to
 
-The end of the range. This can be a number, a date, or a date expressed as a string.
+(optional) The end of the range. This can be a number, a date, or a date expressed as a string.
 
-## Examples
+## Important notes:
+
+BUCKET can operate in two modes:
+- one where the bucket size is computed based on a bucket count recommendation and a range,
+- and another where the bucket size is provided directly.
 
-BUCKET can operate in two modes: one where the bucket size is computed based on a bucket count recommendation and a range, and another where the bucket size is provided directly.
+When the bucket size is provided directly for time interval,
+it is expressed as a *timespan literal*, e.g.
+- GOOD: `BUCKET(@timestamp, 1 month)`
+- BAD: `BUCKET(@timestamp, "month")`
+
+## Examples
 
 For instance, asking for at most 20 buckets over a year results in monthly buckets:
 

x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt

@@ -16,6 +16,14 @@ This is the interval to which the date will be rounded down. It is expressed usi
 
 This is the date expression that will be rounded down.
 
+## Important notes
+
+The *interval* parameter of DATE_TRUNC is a timespan literal, NOT a string.
+  - GOOD: `DATE_TRUNC(1 year, date)`
+  - BAD: `DATE_TRUNC("year", date)`
+
+When grouping data by time interval, it is recommended to use BUCKET instead of DATE_TRUNC.
+
 ## Examples
 
 The following example rounds down the hire_date to the nearest year:

x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt

@@ -43,7 +43,7 @@ FROM index
 
 ## Literals
 
-ES|QL currently supports numeric and string literals.
+ES|QL currently supports numeric, string and timespan literals.
 
 ### String Literals
 
@@ -93,11 +93,11 @@ These qualifiers are supported:
 - `quarter`/`quarters`/`q`
 - `year`/`years`/`yr`/`y`
 
-Timespan literals are not whitespace sensitive. These expressions are all valid:
-
-- 1day
-- 1 day
-- 1       day
+Timespan literals are not whitespace sensitive, and should not be wrapped with quotes:
+- GOOD: 1day
+- GOOD: 1 day
+- BAD: "day"
+- BAD: "2 days"
 
 ## Example Queries with Timespan Literals
 
@@ -137,6 +137,15 @@ FROM sales
 | SORT week
 ```
 
+4. The same example with BUCKET instead of DATE_TRUNC:
+
+```esql
+FROM sales
+| WHERE @timestamp > NOW() - 1 quarter
+| STATS weekly_sales = SUM(sales_amount) BY week = BUCKET(@timestamp, 1 week)
+| SORT week
+```
+
 5. Retrieve error logs from the last 15 minutes and group by error type:
 
 ```esql

x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts

@@ -29,10 +29,19 @@ export function getSystemMessageInstructions({
     If a tool does not have properties, leave them out.
 
     It is EXTREMELY important that you generate valid JSON between the \`\`\`json and \`\`\` delimiters.
-    
-    You may call them like this.
 
-    Given the following tool:
+    IMPORTANT: make sure you start and end a tool call with the ${TOOL_USE_START} and ${TOOL_USE_END} markers, it MUST
+    be included in the tool call.
+
+    You may call tools like this:
+
+    ${TOOL_USE_START}
+    \`\`\`json
+    ${JSON.stringify({ name: '[name of the tool]', input: { myProperty: 'myValue' } })}
+    \`\`\`\
+    ${TOOL_USE_END}
+
+    For example, given the following tool:
 
     ${JSON.stringify({
       name: 'my_tool',
@@ -55,13 +64,15 @@ export function getSystemMessageInstructions({
     \`\`\`\
     ${TOOL_USE_END}
 
-    Given the following tool:
+    Another example: given the following tool:
+
     ${JSON.stringify({
       name: 'my_tool_without_parameters',
       description: 'A tool to call without parameters',
     })}
 
-    Use it the following way: 
+    Use it the following way:
+
     ${TOOL_USE_START}
     \`\`\`json
     ${JSON.stringify({ name: 'my_tool_without_parameters', input: {} })}
@@ -77,7 +88,7 @@ export function getSystemMessageInstructions({
         ...(fn.parameters ? { parameters: fn.parameters } : {}),
       }))
     )}
-    
+
     `;
   }
 

x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts

@@ -44,7 +44,7 @@ export function parseInlineFunctionCalls({ logger }: { logger: Logger }) {
         logger.debug('Parsing function call:\n' + buffer);
 
         const match = buffer.match(
-          /<\|tool_use_start\|>\s*```json\n?(.*?)(\n```\s*).*<\|tool_use_end\|>/s
+          /<\|tool_use_start\|>\s*```json\n?(.*?)(\n?```\s*).*<\|tool_use_end\|>/s
         );
 
         const functionCallBody = match?.[1];

x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts

@@ -110,7 +110,7 @@ export class KibanaClient {
     const url = format({
       ...parsed,
       pathname: `/${[
-        baseUrl,
+        ...(baseUrl ? [baseUrl] : []),
         ...(props.ignoreSpaceId || !this.spaceId ? [] : ['s', this.spaceId]),
         props.pathname.startsWith('/') ? props.pathname.substring(1) : props.pathname,
       ].join('/')}`,