From b7dcb9cbb0a4fac711e828dd2cee91a4b17c935d Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Thu, 19 Sep 2024 22:22:44 +0200 Subject: [PATCH] [inference] improve simulated function calling instructions (#193414) ## Summary (and do it also for o11y) (cherry picked from commit a9402939f378bfd9ba6136b9bba687430c64ca32) --- .../inference/scripts/util/kibana_client.ts | 2 ++ .../get_system_instructions.ts | 17 +++++++++++--- .../parse_inline_function_calls.ts | 2 +- .../nl_to_esql/esql_docs/esql-bucket.txt | 19 +++++++++++---- .../nl_to_esql/esql_docs/esql-date_trunc.txt | 8 +++++++ .../nl_to_esql/esql_docs/esql-syntax.txt | 21 ++++++++++++----- .../get_system_message_instructions.ts | 23 ++++++++++++++----- .../parse_inline_function_calls.ts | 2 +- .../scripts/evaluation/kibana_client.ts | 2 +- 9 files changed, 73 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/inference/scripts/util/kibana_client.ts b/x-pack/plugins/inference/scripts/util/kibana_client.ts index f8e96cfcf4ae3..ca26ef76b2c72 100644 --- a/x-pack/plugins/inference/scripts/util/kibana_client.ts +++ b/x-pack/plugins/inference/scripts/util/kibana_client.ts @@ -179,6 +179,7 @@ export class KibanaClient { system, toolChoice, tools, + functionCalling, }) => { const body: ChatCompleteRequestBody = { connectorId: chatCompleteConnectorId, @@ -186,6 +187,7 @@ export class KibanaClient { messages, toolChoice, tools, + functionCalling, }; return stream( diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts index 872e842e03f86..848b8d1f44f07 100644 --- a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts @@ -29,9 +29,18 @@ export function getSystemMessageInstructions({ It is EXTREMELY important that you generate valid JSON between the \`\`\`json and \`\`\` delimiters. - You may call them like this. + IMPORTANT: make sure you start and end a tool call with the ${TOOL_USE_START} and ${TOOL_USE_END} markers, it MUST + be included in the tool call. - Given the following tool: + You may call tools like this: + + ${TOOL_USE_START} + \`\`\`json + ${JSON.stringify({ name: '[name of the tool]', input: { myProperty: 'myValue' } })} + \`\`\`\ + ${TOOL_USE_END} + + For example, given the following tool: ${JSON.stringify({ name: 'my_tool', @@ -54,13 +63,15 @@ export function getSystemMessageInstructions({ \`\`\`\ ${TOOL_USE_END} - Given the following tool: + Another example: given the following tool: + ${JSON.stringify({ name: 'my_tool_without_parameters', description: 'A tool to call without parameters', })} Use it the following way: + ${TOOL_USE_START} \`\`\`json ${JSON.stringify({ name: 'my_tool_without_parameters', input: {} })} diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts index 2fa9dd899e986..3436d7a7edac5 100644 --- a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts @@ -45,7 +45,7 @@ export function parseInlineFunctionCalls({ logger }: { logger: Logger }) { logger.debug('Parsing function call:\n' + buffer); const match = buffer.match( - /<\|tool_use_start\|>\s*```json\n?(.*?)(\n```\s*).*<\|tool_use_end\|>/s + /<\|tool_use_start\|>\s*```json\n?(.*?)(\n?```\s*).*<\|tool_use_end\|>/s ); const functionCallBody = match?.[1]; diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt index 585a0321ef818..617952666542a 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-bucket.txt @@ -4,7 +4,7 @@ The BUCKET function allows you to create groups of values, known as buckets, fro ## Syntax -`BUCKET(field, buckets, from, to)` +`BUCKET(field, buckets [, from, to])` ### Parameters @@ -18,15 +18,24 @@ The target number of buckets, or the desired bucket size if `from` and `to` para #### from -The start of the range. This can be a number, a date, or a date expressed as a string. +(optional) The start of the range. This can be a number, a date, or a date expressed as a string. #### to -The end of the range. This can be a number, a date, or a date expressed as a string. +(optional) The end of the range. This can be a number, a date, or a date expressed as a string. -## Examples +## Important notes: + +BUCKET can operate in two modes: +- one where the bucket size is computed based on a bucket count recommendation and a range, +- and another where the bucket size is provided directly. -BUCKET can operate in two modes: one where the bucket size is computed based on a bucket count recommendation and a range, and another where the bucket size is provided directly. +When the bucket size is provided directly for time interval, +it is expressed as a *timespan literal*, e.g. +- GOOD: `BUCKET(@timestamp, 1 month)` +- BAD: `BUCKET(@timestamp, "month")` + +## Examples For instance, asking for at most 20 buckets over a year results in monthly buckets: diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt index bd1d4b68043b1..7df5249933821 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-date_trunc.txt @@ -16,6 +16,14 @@ This is the interval to which the date will be rounded down. It is expressed usi This is the date expression that will be rounded down. +## Important notes + +The *interval* parameter of DATE_TRUNC is a timespan literal, NOT a string. + - GOOD: `DATE_TRUNC(1 year, date)` + - BAD: `DATE_TRUNC("year", date)` + +When grouping data by time interval, it is recommended to use BUCKET instead of DATE_TRUNC. + ## Examples The following example rounds down the hire_date to the nearest year: diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt index 8259b8d8d5546..85df775422801 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-syntax.txt @@ -43,7 +43,7 @@ FROM index ## Literals -ES|QL currently supports numeric and string literals. +ES|QL currently supports numeric, string and timespan literals. ### String Literals @@ -93,11 +93,11 @@ These qualifiers are supported: - `quarter`/`quarters`/`q` - `year`/`years`/`yr`/`y` -Timespan literals are not whitespace sensitive. These expressions are all valid: - -- 1day -- 1 day -- 1 day +Timespan literals are not whitespace sensitive, and should not be wrapped with quotes: +- GOOD: 1day +- GOOD: 1 day +- BAD: "day" +- BAD: "2 days" ## Example Queries with Timespan Literals @@ -137,6 +137,15 @@ FROM sales | SORT week ``` +4. The same example with BUCKET instead of DATE_TRUNC: + +```esql +FROM sales +| WHERE @timestamp > NOW() - 1 quarter +| STATS weekly_sales = SUM(sales_amount) BY week = BUCKET(@timestamp, 1 week) +| SORT week +``` + 5. Retrieve error logs from the last 15 minutes and group by error type: ```esql diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts index a409a80716320..c80ac022ca6c0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts @@ -29,10 +29,19 @@ export function getSystemMessageInstructions({ If a tool does not have properties, leave them out. It is EXTREMELY important that you generate valid JSON between the \`\`\`json and \`\`\` delimiters. - - You may call them like this. - Given the following tool: + IMPORTANT: make sure you start and end a tool call with the ${TOOL_USE_START} and ${TOOL_USE_END} markers, it MUST + be included in the tool call. + + You may call tools like this: + + ${TOOL_USE_START} + \`\`\`json + ${JSON.stringify({ name: '[name of the tool]', input: { myProperty: 'myValue' } })} + \`\`\`\ + ${TOOL_USE_END} + + For example, given the following tool: ${JSON.stringify({ name: 'my_tool', @@ -55,13 +64,15 @@ export function getSystemMessageInstructions({ \`\`\`\ ${TOOL_USE_END} - Given the following tool: + Another example: given the following tool: + ${JSON.stringify({ name: 'my_tool_without_parameters', description: 'A tool to call without parameters', })} - Use it the following way: + Use it the following way: + ${TOOL_USE_START} \`\`\`json ${JSON.stringify({ name: 'my_tool_without_parameters', input: {} })} @@ -77,7 +88,7 @@ export function getSystemMessageInstructions({ ...(fn.parameters ? { parameters: fn.parameters } : {}), })) )} - + `; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts index 82a613ef0b4f8..4ae3c5bf746e3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts @@ -44,7 +44,7 @@ export function parseInlineFunctionCalls({ logger }: { logger: Logger }) { logger.debug('Parsing function call:\n' + buffer); const match = buffer.match( - /<\|tool_use_start\|>\s*```json\n?(.*?)(\n```\s*).*<\|tool_use_end\|>/s + /<\|tool_use_start\|>\s*```json\n?(.*?)(\n?```\s*).*<\|tool_use_end\|>/s ); const functionCallBody = match?.[1]; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 8246a9ceae71f..ddfa4c98765f7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -110,7 +110,7 @@ export class KibanaClient { const url = format({ ...parsed, pathname: `/${[ - baseUrl, + ...(baseUrl ? [baseUrl] : []), ...(props.ignoreSpaceId || !this.spaceId ? [] : ['s', this.spaceId]), props.pathname.startsWith('/') ? props.pathname.substring(1) : props.pathname, ].join('/')}`,