[ES|QL] Runs the _query requests with the version (#180248)

## Summary Due to #179641 ES is going to add a mandatory field `version`. (it is optional for now but they want to make it required for the GA). The version will change in backwards incompatible changes: - syntax changes - functions / commands output changes This is the first PR which is adding the first version manually for now. In #179641 we are exploring how to deal with versioning in kibana. We are going to offer a solution in 8.15. For now we are adding the version hardcoded (otherwise when the field is marked required kibana _query requests will fail. 👮‍♀️ If you are aware of any other application which uses the _query api and is not updated in this PR please let me know --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Apr 9, 2024 · 125c895 · 125c895
1 parent 4b69083
commit 125c895
Show file tree

Hide file tree

Showing 23 changed files with 69 additions and 8 deletions.
diff --git a/packages/kbn-es-types/src/search.ts b/packages/kbn-es-types/src/search.ts
@@ -678,6 +678,7 @@ export interface ESQLSearchParams {
   // https://github.com/elastic/elasticsearch/pull/102767
   // time_zone?: string;
   query: string;
+  version: string;
   filter?: unknown;
   locale?: string;
   dropNullColumns?: boolean;

diff --git a/packages/kbn-esql-utils/constants.ts b/packages/kbn-esql-utils/constants.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// we are expecting to retrieve this from an API instead
+// https://github.com/elastic/elasticsearch/issues/107069
+export const ESQL_LATEST_VERSION = '2024.04.01';
diff --git a/packages/kbn-esql-utils/index.ts b/packages/kbn-esql-utils/index.ts
@@ -17,3 +17,5 @@ export {
   getESQLWithSafeLimit,
   TextBasedLanguages,
 } from './src';
+
+export { ESQL_LATEST_VERSION } from './constants';
diff --git a/packages/kbn-generate-csv/src/generate_csv_esql.test.ts b/packages/kbn-generate-csv/src/generate_csv_esql.test.ts
@@ -22,6 +22,7 @@ import { IKibanaSearchResponse } from '@kbn/data-plugin/common';
 import { IScopedSearchClient } from '@kbn/data-plugin/server';
 import { dataPluginMock } from '@kbn/data-plugin/server/mocks';
 import { CancellationToken } from '@kbn/reporting-common';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import type { ReportingConfigType } from '@kbn/reporting-server';
 import type { ESQLSearchReponse as ESQLSearchResponse } from '@kbn/es-types';
 import {
@@ -311,7 +312,7 @@ describe('CsvESQLGenerator', () => {
       );
 
       expect(mockDataClientSearchFn).toBeCalledWith(
-        { params: { filter: undefined, locale: 'en', query: '' } },
+        { params: { filter: undefined, locale: 'en', query: '', version: ESQL_LATEST_VERSION } },
         {
           strategy: 'esql',
           transport: {
@@ -389,7 +390,7 @@ describe('CsvESQLGenerator', () => {
       );
 
       expect(mockDataClientSearchFn).toBeCalledWith(
-        { params: { filter: undefined, locale: 'en', query: '' } },
+        { params: { filter: undefined, locale: 'en', query: '', version: ESQL_LATEST_VERSION } },
         {
           strategy: 'esql',
           transport: {
@@ -485,6 +486,7 @@ describe('CsvESQLGenerator', () => {
             },
             locale: 'en',
             query: '',
+            version: ESQL_LATEST_VERSION,
           },
         },
         {

diff --git a/packages/kbn-generate-csv/src/generate_csv_esql.ts b/packages/kbn-generate-csv/src/generate_csv_esql.ts
@@ -8,7 +8,7 @@
 
 import { lastValueFrom } from 'rxjs';
 import type { Writable } from 'stream';
-
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { errors as esErrors } from '@elastic/elasticsearch';
 import type { IScopedClusterClient, IUiSettingsClient, Logger } from '@kbn/core/server';
 import {
@@ -100,6 +100,7 @@ export class CsvESQLGenerator {
         // we will need to add it back in once it is supported again.
         // https://github.com/elastic/elasticsearch/pull/102767
         // timezone
+        version: ESQL_LATEST_VERSION,
       },
     };
 

diff --git a/packages/kbn-generate-csv/tsconfig.json b/packages/kbn-generate-csv/tsconfig.json
@@ -28,5 +28,6 @@
     "@kbn/reporting-export-types-csv-common",
     "@kbn/es-query",
     "@kbn/es-types",
+    "@kbn/esql-utils",
   ]
 }
diff --git a/...kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts b/...kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts
@@ -15,7 +15,7 @@ import type {
 } from '@kbn/data-plugin/public';
 import type { ESQLSearchParams, ESQLSearchReponse } from '@kbn/es-types';
 import type { AggregateQuery } from '@kbn/es-query';
-import { getESQLWithSafeLimit } from '@kbn/esql-utils';
+import { ESQL_LATEST_VERSION, getESQLWithSafeLimit } from '@kbn/esql-utils';
 import type { FieldStatsResponse } from '../../types';
 import {
   buildSearchFilter,
@@ -77,6 +77,7 @@ export const loadFieldStatsTextBased: LoadFieldStatsTextBasedHandler = async ({
             params: {
               ...(filter ? { filter } : {}),
               ...body,
+              version: ESQL_LATEST_VERSION,
             },
           },
           {

diff --git a/src/plugins/data/common/search/expressions/esql.ts b/src/plugins/data/common/search/expressions/esql.ts
@@ -17,6 +17,7 @@ import { Observable, defer, throwError } from 'rxjs';
 import { catchError, map, switchMap, tap } from 'rxjs';
 import { buildEsQuery } from '@kbn/es-query';
 import type { ESQLSearchReponse, ESQLSearchParams } from '@kbn/es-types';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { getEsQueryConfig } from '../../es_query';
 import { getTime } from '../../query';
 import {
@@ -130,6 +131,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
             query,
             // time_zone: timezone,
             locale,
+            version: ESQL_LATEST_VERSION,
           };
           if (input) {
             const esQueryConfigs = getEsQueryConfig(

diff --git a/...lugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.test.ts b/...lugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.test.ts
@@ -8,6 +8,7 @@
 
 import { firstValueFrom } from 'rxjs';
 import { KbnServerError } from '@kbn/kibana-utils-plugin/server';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { KbnSearchError } from '../../report_search_error';
 import { errors } from '@elastic/elasticsearch';
 import indexNotFoundException from '../../../../common/search/test_data/index_not_found_exception.json';
@@ -66,6 +67,7 @@ describe('ES|QL async search strategy', () => {
         const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
         const params = {
           query: 'from logs* | limit 10',
+          version: ESQL_LATEST_VERSION,
         };
         await esSearch
           .search(
@@ -89,6 +91,7 @@ describe('ES|QL async search strategy', () => {
 
         const params = {
           query: 'from logs* | limit 10',
+          version: ESQL_LATEST_VERSION,
         };
         const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 
@@ -108,6 +111,7 @@ describe('ES|QL async search strategy', () => {
           query: 'from logs* | limit 10',
           wait_for_completion_timeout: '10s',
           keep_alive: '5m',
+          version: ESQL_LATEST_VERSION,
         };
         const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 
@@ -123,7 +127,7 @@ describe('ES|QL async search strategy', () => {
       it('sets transport options on POST requests', async () => {
         const transportOptions = { maxRetries: 1 };
         mockApiCaller.mockResolvedValueOnce(mockAsyncResponse);
-        const params = { query: 'from logs' };
+        const params = { query: 'from logs', version: ESQL_LATEST_VERSION };
         const esSearch = esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 
         await firstValueFrom(
@@ -140,6 +144,7 @@ describe('ES|QL async search strategy', () => {
               wait_for_completion_timeout: '100ms',
               keep_on_completion: false,
               query: 'from logs',
+              version: ESQL_LATEST_VERSION,
             },
           }),
           expect.objectContaining({ maxRetries: 1, meta: true, signal: undefined })
@@ -150,6 +155,7 @@ describe('ES|QL async search strategy', () => {
         mockApiCaller.mockResolvedValueOnce(mockAsyncResponse);
         const params = {
           query: 'from logs* | limit 10',
+          version: ESQL_LATEST_VERSION,
         };
         const esSearch = esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 
@@ -175,6 +181,7 @@ describe('ES|QL async search strategy', () => {
 
         const params = {
           query: 'from logs* | limit 10',
+          version: ESQL_LATEST_VERSION,
         };
         const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 
@@ -197,6 +204,7 @@ describe('ES|QL async search strategy', () => {
 
         const params = {
           query: 'from logs* | limit 10',
+          version: ESQL_LATEST_VERSION,
         };
         const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
         const abortController = new AbortController();
@@ -230,6 +238,7 @@ describe('ES|QL async search strategy', () => {
 
       const params = {
         query: 'from logs* | limit 10',
+        version: ESQL_LATEST_VERSION,
       };
       const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 
@@ -253,6 +262,7 @@ describe('ES|QL async search strategy', () => {
 
       const params = {
         query: 'from logs* | limit 10',
+        version: ESQL_LATEST_VERSION,
       };
       const esSearch = await esqlAsyncSearchStrategyProvider(mockSearchConfig, mockLogger);
 

diff --git a/src/plugins/data/tsconfig.json b/src/plugins/data/tsconfig.json
@@ -53,7 +53,8 @@
     "@kbn/bfetch-error",
     "@kbn/es-types",
     "@kbn/code-editor",
-    "@kbn/core-test-helpers-model-versions"
+    "@kbn/core-test-helpers-model-versions",
+    "@kbn/esql-utils"
   ],
   "exclude": [
     "target/**/*",

diff --git a/test/api_integration/apis/esql/errors.ts b/test/api_integration/apis/esql/errors.ts
@@ -11,6 +11,7 @@ import Path from 'path';
 import expect from '@kbn/expect';
 import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
 import { REPO_ROOT } from '@kbn/repo-info';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import uniqBy from 'lodash/uniqBy';
 import { groupBy, mapValues } from 'lodash';
 import { FtrProviderContext } from '../../ftr_provider_context';
@@ -123,6 +124,7 @@ export default function ({ getService }: FtrProviderContext) {
         path: '/_query',
         body: {
           query,
+          version: ESQL_LATEST_VERSION,
         },
       });
       return { resp, error: undefined };

diff --git a/test/tsconfig.json b/test/tsconfig.json
@@ -73,5 +73,6 @@
     "@kbn/links-plugin",
     "@kbn/ftr-common-functional-ui-services",
     "@kbn/monaco",
+    "@kbn/esql-utils",
   ]
 }
diff --git a/...alizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts b/...alizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts
@@ -14,7 +14,7 @@ import { type UseCancellableSearch, useCancellableSearch } from '@kbn/ml-cancell
 import type { estypes } from '@elastic/elasticsearch';
 import type { ISearchOptions } from '@kbn/data-plugin/common';
 import type { TimeBucketsInterval } from '@kbn/ml-time-buckets';
-import { getESQLWithSafeLimit } from '@kbn/esql-utils';
+import { getESQLWithSafeLimit, ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { OMIT_FIELDS } from '../../../../../common/constants';
 import type {
   DataStatsFetchProgress,
@@ -84,6 +84,7 @@ const getESQLDocumentCountStats = async (
       params: {
         query: esqlBaseQuery + aggQuery,
         ...(filter ? { filter } : {}),
+        version: ESQL_LATEST_VERSION,
       },
     };
     try {
@@ -129,6 +130,7 @@ const getESQLDocumentCountStats = async (
       params: {
         query: esqlBaseQuery + ' | STATS _count_ = COUNT(*)  | LIMIT 1',
         ...(filter ? { filter } : {}),
+        version: ESQL_LATEST_VERSION,
       },
     };
     try {
@@ -247,6 +249,7 @@ export const useESQLOverallStatsData = (
             params: {
               query: esqlBaseQuery + '| LIMIT 0',
               ...(filter ? { filter } : {}),
+              version: ESQL_LATEST_VERSION,
             },
           },
           { strategy: ESQL_SEARCH_STRATEGY }

diff --git a/...lication/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts b/...lication/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts
@@ -9,6 +9,7 @@ import pLimit from 'p-limit';
 import { chunk } from 'lodash';
 import { isDefined } from '@kbn/ml-is-defined';
 import type { ESQLSearchReponse } from '@kbn/es-types';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { i18n } from '@kbn/i18n';
@@ -105,6 +106,7 @@ const getESQLOverallStatsInChunk = async ({
       params: {
         query,
         ...(filter ? { filter } : {}),
+        version: ESQL_LATEST_VERSION,
       },
     };
 

diff --git a/...c/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts b/...c/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts
@@ -8,6 +8,7 @@
 import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
 import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
 import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
 import { getSafeESQLName } from '../requests/esql_utils';
 import type { DateFieldStats, FieldStatsError } from '../../../../../common/types/field_stats';
@@ -40,6 +41,7 @@ export const getESQLDateFieldStats = async ({
       params: {
         query: esqlBaseQuery + dateStatsQuery,
         ...(filter ? { filter } : {}),
+        version: ESQL_LATEST_VERSION,
       },
     };
     try {

diff --git a/...pplication/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts b/...pplication/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts
@@ -8,6 +8,7 @@
 import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
 import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
 import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { chunk } from 'lodash';
 import pLimit from 'p-limit';
 import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
@@ -69,6 +70,7 @@ const getESQLNumericFieldStatsInChunk = async ({
       params: {
         query: esqlBaseQuery + numericStatsQuery,
         ...(filter ? { filter } : {}),
+        version: ESQL_LATEST_VERSION,
       },
     };
     try {

diff --git a/...c/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts b/...c/application/index_data_visualizer/search_strategy/esql_requests/get_text_field_stats.ts
@@ -8,6 +8,7 @@
 import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
 import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
 import { ESQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
 import type { FieldExamples, FieldStatsError } from '../../../../../common/types/field_stats';
 
@@ -39,6 +40,7 @@ export const getESQLExampleFieldValues = async ({
             `| KEEP ${textFields.map((f) => f.name).join(',')}
            | LIMIT 10`,
           ...(filter ? { filter } : {}),
+          version: ESQL_LATEST_VERSION,
         },
       };
       const textFieldsResp = await runRequest(request, { strategy: ESQL_SEARCH_STRATEGY });

diff --git a/x-pack/plugins/maps/public/classes/sources/esql_source/esql_source.tsx b/x-pack/plugins/maps/public/classes/sources/esql_source/esql_source.tsx
@@ -11,7 +11,11 @@ import { lastValueFrom } from 'rxjs';
 import { tap } from 'rxjs';
 import { v4 as uuidv4 } from 'uuid';
 import { Adapters } from '@kbn/inspector-plugin/common/adapters';
-import { getIndexPatternFromESQLQuery, getLimitFromESQLQuery } from '@kbn/esql-utils';
+import {
+  getIndexPatternFromESQLQuery,
+  getLimitFromESQLQuery,
+  ESQL_LATEST_VERSION,
+} from '@kbn/esql-utils';
 import { buildEsQuery } from '@kbn/es-query';
 import type { Filter, Query } from '@kbn/es-query';
 import type { ESQLSearchParams, ESQLSearchReponse } from '@kbn/es-types';
@@ -153,6 +157,7 @@ export class ESQLSource
     const params: ESQLSearchParams = {
       query: this._descriptor.esql,
       dropNullColumns: true,
+      version: ESQL_LATEST_VERSION,
     };
 
     const query: Query[] = [];

diff --git a/...ins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts b/...ins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts
@@ -11,6 +11,7 @@ import pLimit from 'p-limit';
 import Path from 'path';
 import { lastValueFrom, startWith } from 'rxjs';
 import { promisify } from 'util';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { FunctionVisibility, MessageRole } from '@kbn/observability-ai-assistant-plugin/common';
 import {
   VisualizeESQLUserIntention,
@@ -95,6 +96,7 @@ export function registerQueryFunction({
         path: '_query',
         body: {
           query,
+          version: ESQL_LATEST_VERSION,
         },
       });
 

diff --git a/.../observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts b/.../observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts
@@ -6,6 +6,7 @@
  */
 import { esFieldTypeToKibanaFieldType } from '@kbn/field-types';
 import type { ESQLSearchReponse } from '@kbn/es-types';
+import { ESQL_LATEST_VERSION } from '@kbn/esql-utils';
 import { VisualizeESQLUserIntention } from '@kbn/observability-ai-assistant-plugin/common/functions/visualize_esql';
 import { visualizeESQLFunction } from '../../common/functions/visualize_esql';
 import { FunctionRegistrationParameters } from '.';
@@ -28,6 +29,7 @@ export function registerVisualizeESQLFunction({
         path: '_query',
         body: {
           query: performantQuery,
+          version: ESQL_LATEST_VERSION,
         },
       })) as ESQLSearchReponse;
       const columns =