From 6b778cb071fa8b0555f51bc7aa721e1947474b07 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 24 Jun 2024 16:53:46 -0700 Subject: [PATCH 1/7] Improve SearchSource SearchRequest type --- .../src/lib/search_cursor_pit.ts | 8 +--- .../src/lib/search_cursor_scroll.ts | 8 +--- .../search_source/fetch/get_search_params.ts | 7 ++- .../search/search_source/fetch/types.ts | 9 +++- .../search/search_source/query_to_fields.ts | 8 ++-- .../search_source/search_source.test.ts | 19 +++++--- .../search/search_source/search_source.ts | 48 +++++++++++-------- .../vega/public/data_model/search_api.ts | 2 +- .../public/datasources/form_based/utils.tsx | 3 +- x-pack/plugins/lens/public/types.ts | 3 +- .../services/actions/actions_audit_log.ts | 37 +++++++------- .../factory/risk_score/all/index.ts | 4 +- 12 files changed, 84 insertions(+), 72 deletions(-) diff --git a/packages/kbn-generate-csv/src/lib/search_cursor_pit.ts b/packages/kbn-generate-csv/src/lib/search_cursor_pit.ts index db25be3488830..902880416e832 100644 --- a/packages/kbn-generate-csv/src/lib/search_cursor_pit.ts +++ b/packages/kbn-generate-csv/src/lib/search_cursor_pit.ts @@ -9,11 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { Logger } from '@kbn/core/server'; import { lastValueFrom } from 'rxjs'; -import { - ES_SEARCH_STRATEGY, - type ISearchSource, - type SearchRequest, -} from '@kbn/data-plugin/common'; +import { ES_SEARCH_STRATEGY, type ISearchSource } from '@kbn/data-plugin/common'; import { SearchCursor, type SearchCursorClients, type SearchCursorSettings } from './search_cursor'; import { i18nTexts } from './i18n_texts'; @@ -74,7 +70,7 @@ export class SearchCursorPit extends SearchCursor { return pitId; } - protected async searchWithPit(searchBody: SearchRequest) { + protected async searchWithPit(searchBody: estypes.SearchRequest) { const { maxConcurrentShardRequests, scroll, taskInstanceFields } = this.settings; // maxConcurrentShardRequests=0 is not supported diff --git a/packages/kbn-generate-csv/src/lib/search_cursor_scroll.ts b/packages/kbn-generate-csv/src/lib/search_cursor_scroll.ts index 24fdd10d59d2a..75181209593d3 100644 --- a/packages/kbn-generate-csv/src/lib/search_cursor_scroll.ts +++ b/packages/kbn-generate-csv/src/lib/search_cursor_scroll.ts @@ -10,11 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import { lastValueFrom } from 'rxjs'; import type { Logger } from '@kbn/core/server'; import type { IEsSearchResponse } from '@kbn/search-types'; -import { - ES_SEARCH_STRATEGY, - type ISearchSource, - type SearchRequest, -} from '@kbn/data-plugin/common'; +import { ES_SEARCH_STRATEGY, type ISearchSource } from '@kbn/data-plugin/common'; import { SearchCursor, type SearchCursorClients, type SearchCursorSettings } from './search_cursor'; import { i18nTexts } from './i18n_texts'; @@ -32,7 +28,7 @@ export class SearchCursorScroll extends SearchCursor { // The first search query begins the scroll context in ES public async initialize() {} - private async scan(searchBody: SearchRequest) { + private async scan(searchBody: estypes.SearchRequest) { const { includeFrozen, maxConcurrentShardRequests, scroll, taskInstanceFields } = this.settings; // maxConcurrentShardRequests=0 is not supported diff --git a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts index 712f0d4192edc..6c882678cf0c4 100644 --- a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts +++ b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts @@ -37,12 +37,15 @@ export function getSearchParamsFromRequest( const searchParams = getSearchParams(getConfig); // eslint-disable-next-line @typescript-eslint/naming-convention const { track_total_hits, ...body } = searchRequest.body; + const dataView = typeof searchRequest.index !== 'string' ? searchRequest.index : undefined; + const index = dataView?.title ?? `${searchRequest.index}`; return { - index: searchRequest.index.title || searchRequest.index, + index, body, + // @ts-ignore track_total_hits, - ...(searchRequest.index?.allowHidden && { expand_wildcards: 'all' }), + ...(dataView?.getAllowHidden() && { expand_wildcards: 'all' }), ...searchParams, }; } diff --git a/src/plugins/data/common/search/search_source/fetch/types.ts b/src/plugins/data/common/search/search_source/fetch/types.ts index b798bef9b859c..e94f37193239d 100644 --- a/src/plugins/data/common/search/search_source/fetch/types.ts +++ b/src/plugins/data/common/search/search_source/fetch/types.ts @@ -6,6 +6,8 @@ * Side Public License, v 1. */ import type { IKibanaSearchResponse } from '@kbn/search-types'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import type { AggregateQuery, Filter, Query } from '@kbn/es-query'; import { SearchSourceSearchOptions } from '../../..'; import { GetConfigFn } from '../../../types'; @@ -17,7 +19,12 @@ import { GetConfigFn } from '../../../types'; * where `ISearchRequestParams` is used externally instead. * FIXME: replace with estypes.SearchRequest? */ -export type SearchRequest = Record; +export interface SearchRequest { + index?: DataView | string; + query?: Array; + filters?: Filter[] | (() => Filter[]); + [propName: string]: any; +} export interface FetchHandlers { getConfig: GetConfigFn; diff --git a/src/plugins/data/common/search/search_source/query_to_fields.ts b/src/plugins/data/common/search/search_source/query_to_fields.ts index ebae3354dcea9..c25fbd3481620 100644 --- a/src/plugins/data/common/search/search_source/query_to_fields.ts +++ b/src/plugins/data/common/search/search_source/query_to_fields.ts @@ -7,7 +7,7 @@ */ import { DataViewLazy } from '@kbn/data-views-plugin/common'; -import { fromKueryExpression, getKqlFieldNames } from '@kbn/es-query'; +import { fromKueryExpression, getKqlFieldNames, isFilter, isOfQueryType } from '@kbn/es-query'; import type { SearchRequest } from './fetch'; import { EsQuerySortValue } from '../..'; @@ -25,21 +25,21 @@ export async function queryToFields({ const sortArr = Array.isArray(sort) ? sort : [sort]; fields.push(...sortArr.flatMap((s) => Object.keys(s))); } - for (const query of request.query) { + for (const query of (request.query ?? []).filter(isOfQueryType)) { if (query.query && query.language === 'kuery') { const nodes = fromKueryExpression(query.query); const queryFields = getKqlFieldNames(nodes); fields = fields.concat(queryFields); } } - const filters = request.filters; + const { filters } = request; if (filters) { const filtersArr = Array.isArray(filters) ? filters : [filters]; for (const f of filtersArr) { // unified search bar filters have meta object and key (regular filters) // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is) // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all) - if (f?.meta?.key && f.meta.disabled !== true) { + if (isFilter(f) && f?.meta?.key && f.meta.disabled !== true) { fields.push(f.meta.key); } } diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 7d2a3d6148c01..ef8341c3d2c5d 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -36,12 +36,14 @@ const indexPattern = { fields: [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }, { name: '_id' }], getComputedFields, getSourceFiltering: () => mockSource, + getAllowHidden: jest.fn(), } as unknown as DataView; const indexPattern2 = { title: 'foo', getComputedFields, getSourceFiltering: () => mockSource2, + getAllowHidden: jest.fn(), } as unknown as DataView; const fields3 = [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }]; @@ -1474,10 +1476,15 @@ describe('SearchSource', () => { return buildExpression(ast).toString(); } + beforeEach(() => { + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + }); + test('should generate an expression AST', () => { expect(toString(searchSource.toExpressionAst())).toMatchInlineSnapshot(` "kibana_context - | esdsl dsl=\\"{}\\"" + | esdsl dsl=\\"{}\\" index=\\"1234\\"" `); }); @@ -1486,7 +1493,7 @@ describe('SearchSource', () => { expect(toString(searchSource.toExpressionAst())).toMatchInlineSnapshot(` "kibana_context q={kql q=\\"something\\"} - | esdsl dsl=\\"{}\\"" + | esdsl dsl=\\"{}\\" index=\\"1234\\"" `); }); @@ -1504,7 +1511,7 @@ describe('SearchSource', () => { expect(toString(searchSource.toExpressionAst())).toMatchInlineSnapshot(` "kibana_context filters={kibanaFilter query=\\"{\\\\\\"query_string\\\\\\":{\\\\\\"query\\\\\\":\\\\\\"query1\\\\\\"}}\\"} filters={kibanaFilter query=\\"{\\\\\\"query_string\\\\\\":{\\\\\\"query\\\\\\":\\\\\\"query2\\\\\\"}}\\"} - | esdsl dsl=\\"{}\\"" + | esdsl dsl=\\"{}\\" index=\\"1234\\"" `); }); @@ -1517,7 +1524,7 @@ describe('SearchSource', () => { expect(toString(searchSource.toExpressionAst())).toMatchInlineSnapshot(` "kibana_context filters={kibanaFilter query=\\"{\\\\\\"query_string\\\\\\":{\\\\\\"query\\\\\\":\\\\\\"query\\\\\\"}}\\"} - | esdsl dsl=\\"{}\\"" + | esdsl dsl=\\"{}\\" index=\\"1234\\"" `); }); @@ -1540,7 +1547,7 @@ describe('SearchSource', () => { expect(toString(childSearchSource.toExpressionAst())).toMatchInlineSnapshot(` "kibana_context q={kql q=\\"something2\\"} q={kql q=\\"something1\\"} filters={kibanaFilter query=\\"{\\\\\\"query_string\\\\\\":{\\\\\\"query\\\\\\":\\\\\\"query2\\\\\\"}}\\"} filters={kibanaFilter query=\\"{\\\\\\"query_string\\\\\\":{\\\\\\"query\\\\\\":\\\\\\"query1\\\\\\"}}\\"} - | esdsl dsl=\\"{}\\"" + | esdsl dsl=\\"{}\\" index=\\"1234\\"" `); }); @@ -1558,7 +1565,7 @@ describe('SearchSource', () => { expect(toString(searchSource.toExpressionAst())).toMatchInlineSnapshot(` "kibana_context - | esdsl size=1000 dsl=\\"{}\\"" + | esdsl size=1000 dsl=\\"{}\\" index=\\"1234\\"" `); }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index a5009f350bd3a..fcb2778e72f0e 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -649,9 +649,12 @@ export class SearchSource { switch (key) { case 'filter': - return addToRoot('filters', (data.filters || []).concat(val)); + return addToRoot( + 'filters', + (typeof data.filters === 'function' ? data.filters() : data.filters || []).concat(val) + ); case 'query': - return addToRoot(key, (data[key] || []).concat(val)); + return addToRoot(key, (data.query || []).concat(val)); case 'fields': // This will pass the passed in parameters to the new fields API. // Also if will only return scripted fields that are part of the specified @@ -662,7 +665,7 @@ export class SearchSource { return addToBody('fields', val); case 'fieldsFromSource': // preserves legacy behavior - const fields = [...new Set((data[key] || []).concat(val))]; + const fields = [...new Set((data.fieldsFromSource || []).concat(val))]; return addToRoot(key, fields); case 'index': case 'type': @@ -709,8 +712,10 @@ export class SearchSource { return searchRequest; } - private getIndexType(index?: DataView) { - return this.shouldOverwriteDataViewType ? this.overwriteDataViewType : index?.type; + private getIndexType(index?: DataView | string) { + return this.shouldOverwriteDataViewType + ? this.overwriteDataViewType + : typeof index !== 'string' && index?.type; } private readonly getFieldName = (fld: SearchFieldValue): string => @@ -782,25 +787,24 @@ export class SearchSource { private flatten() { const { getConfig } = this.dependencies; - const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? []; + const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? []; const searchRequest = this.mergeProps(); searchRequest.body = searchRequest.body || {}; const { body, index } = searchRequest; + const dataView = typeof index !== 'string' ? index : undefined; // get some special field types from the index pattern - const { docvalueFields, scriptFields, runtimeFields } = index - ? index.getComputedFields() - : { - docvalueFields: [], - scriptFields: {}, - runtimeFields: {}, - }; + const { docvalueFields, scriptFields, runtimeFields } = dataView?.getComputedFields() ?? { + docvalueFields: [], + scriptFields: {}, + runtimeFields: {}, + }; const fieldListProvided = !!body.fields; // set defaults const _source = - index && !body.hasOwnProperty('_source') ? index.getSourceFiltering() : body._source; + index && !body.hasOwnProperty('_source') ? dataView?.getSourceFiltering() : body._source; // get filter if data view specified, otherwise null filter const filter = this.getFieldFilter({ bodySourceExcludes: _source?.excludes, metaFields }); @@ -892,7 +896,7 @@ export class SearchSource { runtime_mappings: runtimeFields, script_fields: scriptedFields, fields: this.getFieldsList({ - index, + index: dataView, fields, docvalueFields: body.docvalue_fields, fieldsFromSource, @@ -960,7 +964,12 @@ export class SearchSource { ...getEsQueryConfig({ get: getConfig }), filtersInMustClause, }; - return buildEsQuery(index, query, filters, esQueryConfigs); + return buildEsQuery( + typeof index !== 'string' ? index : undefined, + query, + typeof filters === 'function' ? filters() : filters, + esQueryConfigs + ); } private getRemainingFields({ @@ -1157,6 +1166,7 @@ export class SearchSource { toExpressionAst({ asDatatable = true }: ExpressionAstOptions = {}): ExpressionAstExpression { const searchRequest = this.mergeProps(); const { body, index, query } = searchRequest; + const dataView = typeof index !== 'string' ? index : undefined; const filters = ( typeof searchRequest.filters === 'function' ? searchRequest.filters() : searchRequest.filters @@ -1164,7 +1174,7 @@ export class SearchSource { const ast = buildExpression([ buildExpressionFunction('kibana_context', { - q: query?.map(queryToAst), + q: query?.filter(isOfQueryType).map(queryToAst), filters: filters && filtersToAst(filters), }), ]).toAst(); @@ -1181,7 +1191,7 @@ export class SearchSource { const aggConfigs = aggs instanceof AggConfigs ? aggs - : index && aggs && this.dependencies.aggs.createAggConfigs(index, aggs); + : dataView && aggs && this.dependencies.aggs.createAggConfigs(dataView, aggs); if (aggConfigs) { ast.chain.push(...aggConfigs.toExpressionAst().chain); @@ -1190,7 +1200,7 @@ export class SearchSource { buildExpressionFunction('esdsl', { size: body?.size, dsl: JSON.stringify({}), - index: index?.id, + index: typeof index === 'string' ? index : `${dataView?.id}`, }).toAst() ); } diff --git a/src/plugins/vis_types/vega/public/data_model/search_api.ts b/src/plugins/vis_types/vega/public/data_model/search_api.ts index b15e5a9bda5ca..c584407f0cc40 100644 --- a/src/plugins/vis_types/vega/public/data_model/search_api.ts +++ b/src/plugins/vis_types/vega/public/data_model/search_api.ts @@ -72,7 +72,7 @@ export class SearchAPI { }); return from( - extendSearchParamsWithRuntimeFields(indexPatterns, requestParams, request.index) + extendSearchParamsWithRuntimeFields(indexPatterns, requestParams, `${request.index}`) ).pipe( tap((params) => { /** inspect request data **/ diff --git a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx index 8d74c44c40db6..d9216f2a56d85 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx @@ -17,7 +17,6 @@ import { EuiLink, EuiSpacer } from '@elastic/eui'; import type { DatatableColumn } from '@kbn/expressions-plugin/common'; import { groupBy, escape, uniq, uniqBy } from 'lodash'; import type { Query } from '@kbn/data-plugin/common'; -import { SearchRequest } from '@kbn/data-plugin/common'; import { type SearchResponseWarning, @@ -269,7 +268,7 @@ const accuracyModeEnabledWarning = ( export function getSearchWarningMessages( state: FormBasedPersistedState, warning: SearchResponseWarning, - request: SearchRequest, + request: estypes.SearchRequest, response: estypes.SearchResponse, theme: ThemeServiceStart ): UserMessage[] { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index c1ef27f63100d..06863e688faa9 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -39,7 +39,6 @@ import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { FieldFormatParams } from '@kbn/field-formats-plugin/common'; import type { SearchResponseWarning } from '@kbn/search-response-warnings'; import type { EuiButtonIconProps } from '@elastic/eui'; -import { SearchRequest } from '@kbn/data-plugin/public'; import { estypes } from '@elastic/elasticsearch'; import React from 'react'; import { CellValueContext } from '@kbn/embeddable-plugin/public'; @@ -483,7 +482,7 @@ export interface Datasource { getSearchWarningMessages?: ( state: P, warning: SearchResponseWarning, - request: SearchRequest, + request: estypes.SearchRequest, response: estypes.SearchResponse ) => UserMessage[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts index e53156cee78ea..aa31119227099 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts @@ -6,9 +6,8 @@ */ import type { Logger } from '@kbn/core/server'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; import type { TransportResult } from '@elastic/elasticsearch'; -import type { SearchRequest } from '@kbn/data-plugin/common'; import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common'; import { ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, @@ -203,24 +202,22 @@ const getActionRequestsResult = async ({ indexName: ENDPOINT_ACTIONS_INDEX, }); - const actionsSearchQuery: SearchRequest = { + const actionsSearchQuery: estypes.SearchRequest = { index: hasLogsEndpointActionsIndex ? ACTION_REQUEST_INDICES : AGENT_ACTIONS_INDEX, size, from, - body: { - query: { - bool: { - filter: actionsFilters, - }, + query: { + bool: { + filter: actionsFilters, }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], }, + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], }; let actionRequests: TransportResult, unknown>; @@ -268,16 +265,14 @@ const getActionResponsesResult = async ({ indexName: ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, }); - const responsesSearchQuery: SearchRequest = { + const responsesSearchQuery: estypes.SearchRequest = { index: hasLogsEndpointActionResponsesIndex ? ACTION_RESPONSE_INDICES : AGENT_ACTIONS_RESULTS_INDEX, size: 1000, - body: { - query: { - bool: { - filter: responsesFilters, - }, + query: { + bool: { + filter: responsesFilters, }, }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score/all/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score/all/index.ts index 7a72ae7f60680..395e337fbce7a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score/all/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score/all/index.ts @@ -5,10 +5,10 @@ * 2.0. */ import type { IEsSearchResponse } from '@kbn/search-types'; -import type { SearchRequest, TimeRange } from '@kbn/data-plugin/common'; +import type { TimeRange } from '@kbn/data-plugin/common'; import { get, getOr } from 'lodash/fp'; import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server'; -import type { AggregationsMinAggregate } from '@elastic/elasticsearch/lib/api/types'; +import type { AggregationsMinAggregate, SearchRequest } from '@elastic/elasticsearch/lib/api/types'; import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import type { SecuritySolutionFactory } from '../../types'; import type { From be1bc30892af8206a8d37190601fde080a8557bb Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 25 Jun 2024 08:57:07 -0700 Subject: [PATCH 2/7] Fix data view mock --- packages/kbn-discover-utils/src/__mocks__/data_view.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-discover-utils/src/__mocks__/data_view.ts b/packages/kbn-discover-utils/src/__mocks__/data_view.ts index 99d05640d270b..69375e445c339 100644 --- a/packages/kbn-discover-utils/src/__mocks__/data_view.ts +++ b/packages/kbn-discover-utils/src/__mocks__/data_view.ts @@ -126,6 +126,7 @@ export const buildDataViewMock = ({ return dataViewFields.find((field) => field.name === timeFieldName); }, getRuntimeField: () => null, + getAllowHidden: () => false, } as unknown as DataView; dataView.isTimeBased = () => !!timeFieldName; From 894c72c7e7851fb5ed7a0f1283d53f3b35b27c24 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Tue, 25 Jun 2024 12:18:53 -0700 Subject: [PATCH 3/7] Update snapshots --- .../__snapshots__/data_table_columns.test.tsx.snap | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap b/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap index 7a2a8058d10f0..7513096f49685 100644 --- a/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap +++ b/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap @@ -159,6 +159,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -430,6 +431,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -686,6 +688,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -929,6 +932,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -1190,6 +1194,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -1471,6 +1476,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -1745,6 +1751,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -1900,6 +1907,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -2060,6 +2068,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -2274,6 +2283,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -2474,6 +2484,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -2676,6 +2687,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -2915,6 +2927,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ @@ -3132,6 +3145,7 @@ Array [ "type": "number", }, ], + "getAllowHidden": [Function], "getComputedFields": [Function], "getFieldByName": [MockFunction] { "calls": Array [ From f4bd2e9e410e6c5169a5d8b61a2c49b7e5c2b065 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Wed, 26 Jun 2024 13:55:58 -0700 Subject: [PATCH 4/7] Update query_to_fields filters usage --- .../search/search_source/query_to_fields.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/plugins/data/common/search/search_source/query_to_fields.ts b/src/plugins/data/common/search/search_source/query_to_fields.ts index c25fbd3481620..be6a5982d13de 100644 --- a/src/plugins/data/common/search/search_source/query_to_fields.ts +++ b/src/plugins/data/common/search/search_source/query_to_fields.ts @@ -33,15 +33,12 @@ export async function queryToFields({ } } const { filters } = request; - if (filters) { - const filtersArr = Array.isArray(filters) ? filters : [filters]; - for (const f of filtersArr) { - // unified search bar filters have meta object and key (regular filters) - // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is) - // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all) - if (isFilter(f) && f?.meta?.key && f.meta.disabled !== true) { - fields.push(f.meta.key); - } + for (const f of filters ?? []) { + // unified search bar filters have meta object and key (regular filters) + // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is) + // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all) + if (isFilter(f) && f?.meta?.key && f.meta.disabled !== true) { + fields.push(f.meta.key); } } From ba8353ae5f4bffd9795498e792e9c20a42a93071 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Wed, 26 Jun 2024 14:05:32 -0700 Subject: [PATCH 5/7] Fix filters when they are functions --- .../data/common/search/search_source/query_to_fields.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/common/search/search_source/query_to_fields.ts b/src/plugins/data/common/search/search_source/query_to_fields.ts index be6a5982d13de..a2075366bd390 100644 --- a/src/plugins/data/common/search/search_source/query_to_fields.ts +++ b/src/plugins/data/common/search/search_source/query_to_fields.ts @@ -32,8 +32,8 @@ export async function queryToFields({ fields = fields.concat(queryFields); } } - const { filters } = request; - for (const f of filters ?? []) { + const { filters = [] } = request; + for (const f of typeof filters === 'function' ? filters() : filters) { // unified search bar filters have meta object and key (regular filters) // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is) // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all) From 6fd54cf53287fb5ad4f5884a587552a76f2b9c12 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 8 Jul 2024 16:30:20 -0700 Subject: [PATCH 6/7] Review feedback & update one more reference to incorrect search request --- .../server/options_list/options_list_suggestions_route.ts | 2 +- src/plugins/data/common/search/search_source/fetch/types.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/controls/server/options_list/options_list_suggestions_route.ts b/src/plugins/controls/server/options_list/options_list_suggestions_route.ts index 1e1084d757310..50a4be4013ac9 100644 --- a/src/plugins/controls/server/options_list/options_list_suggestions_route.ts +++ b/src/plugins/controls/server/options_list/options_list_suggestions_route.ts @@ -10,10 +10,10 @@ import { Observable } from 'rxjs'; import { schema } from '@kbn/config-schema'; import { CoreSetup, ElasticsearchClient } from '@kbn/core/server'; -import { SearchRequest } from '@kbn/data-plugin/common'; import { getKbnServerError, reportServerError } from '@kbn/kibana-utils-plugin/server'; import { PluginSetup as UnifiedSearchPluginSetup } from '@kbn/unified-search-plugin/server'; +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { OptionsListRequestBody, OptionsListResponse } from '../../common/options_list/types'; import { getValidationAggregationBuilder } from './options_list_validation_queries'; import { getSuggestionAggregationBuilder } from './suggestion_queries'; diff --git a/src/plugins/data/common/search/search_source/fetch/types.ts b/src/plugins/data/common/search/search_source/fetch/types.ts index e94f37193239d..c7ebd3273034d 100644 --- a/src/plugins/data/common/search/search_source/fetch/types.ts +++ b/src/plugins/data/common/search/search_source/fetch/types.ts @@ -19,12 +19,11 @@ import { GetConfigFn } from '../../../types'; * where `ISearchRequestParams` is used externally instead. * FIXME: replace with estypes.SearchRequest? */ -export interface SearchRequest { +export type SearchRequest = Record> = { index?: DataView | string; query?: Array; filters?: Filter[] | (() => Filter[]); - [propName: string]: any; -} +} & Omit; export interface FetchHandlers { getConfig: GetConfigFn; From 8097a770c26919cb02eda72692bcba78c9c185ed Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Thu, 11 Jul 2024 15:32:59 -0700 Subject: [PATCH 7/7] Review feedback --- .../common/search/search_source/search_source.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index fcb2778e72f0e..c0dc5598e9067 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -715,7 +715,11 @@ export class SearchSource { private getIndexType(index?: DataView | string) { return this.shouldOverwriteDataViewType ? this.overwriteDataViewType - : typeof index !== 'string' && index?.type; + : this.getDataView(index)?.type; + } + + private getDataView(index?: DataView | string): DataView | undefined { + return typeof index !== 'string' ? index : undefined; } private readonly getFieldName = (fld: SearchFieldValue): string => @@ -792,7 +796,7 @@ export class SearchSource { const searchRequest = this.mergeProps(); searchRequest.body = searchRequest.body || {}; const { body, index } = searchRequest; - const dataView = typeof index !== 'string' ? index : undefined; + const dataView = this.getDataView(index); // get some special field types from the index pattern const { docvalueFields, scriptFields, runtimeFields } = dataView?.getComputedFields() ?? { @@ -965,7 +969,7 @@ export class SearchSource { filtersInMustClause, }; return buildEsQuery( - typeof index !== 'string' ? index : undefined, + this.getDataView(index), query, typeof filters === 'function' ? filters() : filters, esQueryConfigs @@ -1166,7 +1170,7 @@ export class SearchSource { toExpressionAst({ asDatatable = true }: ExpressionAstOptions = {}): ExpressionAstExpression { const searchRequest = this.mergeProps(); const { body, index, query } = searchRequest; - const dataView = typeof index !== 'string' ? index : undefined; + const dataView = this.getDataView(index); const filters = ( typeof searchRequest.filters === 'function' ? searchRequest.filters() : searchRequest.filters