From f722117635cfc19f5c08431c75268268a8974c84 Mon Sep 17 00:00:00 2001 From: Cristina Amico Date: Mon, 4 Dec 2023 09:19:26 +0100 Subject: [PATCH 1/2] [Fleet] Replace dataviews suggestions in KQL searchboxes with internal ones (#172190) Fixes https://github.com/elastic/kibana/issues/169760 Related to https://github.com/elastic/kibana/issues/171425 ## Summary [Fleet] Replace dataviews suggestions in KQL searchboxes with internal ones. Now using Fleet mappings to create the suggestions fields instead of fetching them through dataView plugin. This is done for two reasons: - Solves [permission problems](https://github.com/elastic/kibana/issues/169760) when the user doesn't have privileges to read Fleet indices - Allows us to search only those mappings that we want to expose, instead of all of them Only weird thing is that the [querystring component](https://github.com/elastic/kibana/blob/1f8c816901fa286b842ac652b0fce86608d01202/src/plugins/unified_search/public/query_string_input/query_string_input.tsx#L161) has a cap to show max 50 suggestions. Since for agents suggestions we are showing some more fields, so the ones starting with `u` are not visible anymore. I though I had a bug in the way I was creating the `fieldsMap` but in reality there's no way to show more suggestions than 50 (without touching the original component, which I would gladly avoid). ### Screenshots There should be no visible difference with the current suggestions.
Agents ![Screenshot 2023-12-01 at 10 49 55](https://github.com/elastic/kibana/assets/16084106/af73476c-3de2-40c1-93fc-c6a1c28a8a8a) ![Screenshot 2023-12-01 at 10 49 48](https://github.com/elastic/kibana/assets/16084106/5db8b30f-ff9e-4542-a590-f77285dbeef6)
Agent policies ![Screenshot 2023-12-01 at 10 50 09](https://github.com/elastic/kibana/assets/16084106/69756149-6769-48a9-9a34-de482e4e37fc)
Enrollment keys ![Screenshot 2023-12-01 at 10 50 18](https://github.com/elastic/kibana/assets/16084106/e542550a-9721-4f5c-a05b-32829dd8fcee)
### Testing 1. With a normal user, navigate to the "agents", "agent policies" and "enrollment keys" tabs and click on the searchboxes. The suggestions should be visible as normal 2. Create a user with role Fleet "all", Integrations "all". Log in and check the above searchboxes, the suggestions should be visible as normal. Previously they weren't. ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit ad663136c963562abd654e3ab8bce97b752dd3de) # Conflicts: # x-pack/plugins/fleet/common/constants/mappings.ts --- .../plugins/fleet/common/constants/index.ts | 2 + .../{server => common}/constants/mappings.ts | 2 +- .../fleet/components/search_bar.test.tsx | 312 +++++++++++++----- .../fleet/components/search_bar.tsx | 117 ++++--- .../sections/agent_policy/list_page/index.tsx | 1 + .../plugins/fleet/server/constants/index.ts | 2 +- 6 files changed, 309 insertions(+), 127 deletions(-) rename x-pack/plugins/fleet/{server => common}/constants/mappings.ts (99%) diff --git a/x-pack/plugins/fleet/common/constants/index.ts b/x-pack/plugins/fleet/common/constants/index.ts index ca14d612ffea3..3c275f87e1aea 100644 --- a/x-pack/plugins/fleet/common/constants/index.ts +++ b/x-pack/plugins/fleet/common/constants/index.ts @@ -55,3 +55,5 @@ export const FLEET_SERVER_INDICES = [ export const DATA_TIERS = ['data_hot']; export const FLEET_ENROLLMENT_API_PREFIX = 'fleet-enrollment-api-keys'; + +export * from './mappings'; diff --git a/x-pack/plugins/fleet/server/constants/mappings.ts b/x-pack/plugins/fleet/common/constants/mappings.ts similarity index 99% rename from x-pack/plugins/fleet/server/constants/mappings.ts rename to x-pack/plugins/fleet/common/constants/mappings.ts index 9ae9353396803..85ca44dd649d8 100644 --- a/x-pack/plugins/fleet/server/constants/mappings.ts +++ b/x-pack/plugins/fleet/common/constants/mappings.ts @@ -6,7 +6,7 @@ */ /** - * ATTENTION: New mappings for Fleet are defined in the ElasticSearch repo. + * ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo. * * The following mappings declared here closely mirror them * But they are only used to perform validation on the endpoints using ListWithKuery diff --git a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx index 5478aca7c5baa..ebbbeb0814bcf 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx @@ -6,12 +6,20 @@ */ import React from 'react'; +import { act } from '@testing-library/react'; import type { FieldSpec } from '@kbn/data-plugin/common'; import { createFleetTestRendererMock } from '../../../mock'; -import { SearchBar, filterAndConvertFields } from './search_bar'; +import { + AGENTS_PREFIX, + FLEET_ENROLLMENT_API_PREFIX, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + AGENT_POLICY_SAVED_OBJECT_TYPE, +} from '../constants'; + +import { SearchBar, getFieldSpecs } from './search_bar'; const fields = [ { @@ -36,39 +44,6 @@ const fields = [ }, ] as FieldSpec[]; -const allFields = [ - { - name: 'test-index._id', - type: 'string', - esTypes: ['_id'], - }, - { - name: 'test-index.api_key', - type: 'string', - esTypes: ['keyword'], - }, - { - name: 'test-index.name', - type: 'string', - esTypes: ['keyword'], - }, - { - name: 'another-index.version', - type: 'string', - esTypes: ['keyword'], - }, - { - name: 'test2-index.name', - type: 'string', - esTypes: ['keyword'], - }, - { - name: 'fleet-agents.actions', - type: 'string', - esTypes: ['keyword'], - }, -] as FieldSpec[]; - jest.mock('../hooks', () => { return { ...jest.requireActual('../hooks'), @@ -87,23 +62,6 @@ jest.mock('../hooks', () => { }, data: { dataViews: { - getFieldsForWildcard: jest.fn().mockResolvedValue([ - { - name: '_id', - type: 'string', - esTypes: ['_id'], - }, - { - name: 'api_key', - type: 'string', - esTypes: ['keyword'], - }, - { - name: 'name', - type: 'string', - esTypes: ['keyword'], - }, - ]), create: jest.fn().mockResolvedValue({ fields, }), @@ -197,6 +155,9 @@ describe('SearchBar', () => { ); it('renders the search box', async () => { + await act(async () => { + result.queryByTestId('queryInput'); + }); const textArea = result.queryByTestId('queryInput'); expect(textArea).not.toBeNull(); expect(textArea?.getAttribute('placeholder')).toEqual('Filter your data using KQL syntax'); @@ -207,60 +168,243 @@ describe('SearchBar', () => { }); }); -describe('filterAndConvertFields', () => { - it('leaves the fields names unchanged and does not hide any fields if fieldPrefix is not passed', async () => { - expect(filterAndConvertFields(fields, '.test-index')).toEqual({ - _id: { esTypes: ['_id'], name: '_id', type: 'string' }, - api_key: { esTypes: ['keyword'], name: 'api_key', type: 'string' }, - name: { esTypes: ['keyword'], name: 'name', type: 'string' }, - version: { esTypes: ['keyword'], name: 'version', type: 'string' }, - }); +describe('getFieldSpecs', () => { + it('returns fieldSpecs for fleet-agents', () => { + expect(getFieldSpecs(`.${AGENTS_PREFIX}`)).toHaveLength(66); }); - - it('filters out the fields from other indices if indexPattern === .kibana-ingest', async () => { - expect(filterAndConvertFields(allFields, '.kibana_ingest', 'test-index')).toEqual({ - 'test-index._id': { esTypes: ['_id'], name: 'test-index._id', type: 'string' }, - 'test-index.api_key': { esTypes: ['keyword'], name: 'test-index.api_key', type: 'string' }, - 'test-index.name': { esTypes: ['keyword'], name: 'test-index.name', type: 'string' }, - }); + it('returns getFieldSpecs for fleet-enrollment-api-keys', () => { + const indexPattern = `.${FLEET_ENROLLMENT_API_PREFIX}`; + expect(getFieldSpecs(indexPattern)).toHaveLength(8); + expect(getFieldSpecs(indexPattern)).toEqual([ + { + aggregatable: true, + esTypes: ['boolean'], + name: 'active', + searchable: true, + type: 'boolean', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'api_key', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'api_key_id', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['date'], + name: 'created_at', + searchable: true, + type: 'date', + }, + { + aggregatable: true, + esTypes: ['date'], + name: 'expire_at', + searchable: true, + type: 'date', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'name', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'policy_id', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['date'], + name: 'updated_at', + searchable: true, + type: 'date', + }, + ]); }); - it('returns fields unchanged if fieldPrefix and indexPattern are not passed', async () => { - expect(filterAndConvertFields(allFields, undefined, undefined)).toEqual({ - 'another-index.version': { + it('returns getFieldSpecs for fleet-agent-policy', () => { + const indexPattern = `.${AGENT_POLICY_SAVED_OBJECT_TYPE}`; + expect(getFieldSpecs(indexPattern)).toHaveLength(23); + expect(getFieldSpecs(indexPattern)).toEqual([ + { + aggregatable: true, esTypes: ['keyword'], - name: 'another-index.version', + name: 'agent_features.name', + searchable: true, type: 'string', }, - 'fleet-agents.actions': { + { + aggregatable: true, + esTypes: ['boolean'], + name: 'agent_features.enabled', + searchable: true, + type: 'boolean', + }, + { + aggregatable: true, esTypes: ['keyword'], - name: 'fleet-agents.actions', + name: 'data_output_id', + searchable: true, type: 'string', }, - 'test-index._id': { - esTypes: ['_id'], - name: 'test-index._id', + { + aggregatable: true, + esTypes: ['text'], + name: 'description', + searchable: true, type: 'string', }, - 'test-index.api_key': { + { + aggregatable: true, esTypes: ['keyword'], - name: 'test-index.api_key', + name: 'download_source_id', + searchable: true, type: 'string', }, - 'test-index.name': { + { + aggregatable: true, + esTypes: ['keyword'], + name: 'fleet_server_host_id', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['integer'], + name: 'inactivity_timeout', + searchable: true, + type: 'number', + }, + { + aggregatable: true, + esTypes: ['boolean'], + name: 'is_default', + searchable: true, + type: 'boolean', + }, + { + aggregatable: true, + esTypes: ['boolean'], + name: 'is_default_fleet_server', + searchable: true, + type: 'boolean', + }, + { + aggregatable: true, + esTypes: ['boolean'], + name: 'is_managed', + searchable: true, + type: 'boolean', + }, + { + aggregatable: true, esTypes: ['keyword'], - name: 'test-index.name', + name: 'is_preconfigured', + searchable: true, type: 'string', }, - 'test2-index.name': { + { + aggregatable: true, + esTypes: ['boolean'], + name: 'is_protected', + searchable: true, + type: 'boolean', + }, + { + aggregatable: true, esTypes: ['keyword'], - name: 'test2-index.name', + name: 'monitoring_enabled', + searchable: true, type: 'string', }, - }); + { + aggregatable: true, + esTypes: ['false'], + name: 'monitoring_enabled.index', + searchable: true, + type: 'false', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'monitoring_output_id', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'name', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'namespace', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['integer'], + name: 'revision', + searchable: true, + type: 'number', + }, + { + aggregatable: true, + esTypes: ['version'], + name: 'schema_version', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'status', + searchable: true, + type: 'string', + }, + { + aggregatable: true, + esTypes: ['integer'], + name: 'unenroll_timeout', + searchable: true, + type: 'number', + }, + { + aggregatable: true, + esTypes: ['date'], + name: 'updated_at', + searchable: true, + type: 'date', + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'updated_by', + searchable: true, + type: 'string', + }, + ]); }); + expect(getFieldSpecs(`.${PACKAGE_POLICY_SAVED_OBJECT_TYPE}`)).toHaveLength(18); - it('returns empty object if fields is empty', async () => { - expect(filterAndConvertFields([], '.kibana_ingest', 'test-index')).toEqual({}); + it('returns empty array if indexPattern is not one of the previous', async () => { + expect(getFieldSpecs('.kibana_ingest')).toEqual([]); }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx index 5e610a46e3f34..6f23d25f38125 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx @@ -16,9 +16,19 @@ import type { DataView } from '@kbn/data-views-plugin/public'; import { i18n } from '@kbn/i18n'; import { useStartServices } from '../hooks'; -import { INDEX_NAME, AGENTS_PREFIX } from '../constants'; - -const HIDDEN_FIELDS = [`${AGENTS_PREFIX}.actions`, '_id', '_index']; +import { + INDEX_NAME, + AGENTS_PREFIX, + FLEET_ENROLLMENT_API_PREFIX, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + AGENT_POLICY_SAVED_OBJECT_TYPE, +} from '../constants'; +import { + AGENT_POLICY_MAPPINGS, + PACKAGE_POLICIES_MAPPINGS, + AGENT_MAPPINGS, + ENROLLMENT_API_KEY_MAPPINGS, +} from '../../../../common/constants'; const NoWrapQueryStringInput = styled(QueryStringInput)` .kbnQueryBar__textarea { @@ -35,39 +45,69 @@ interface Props { dataTestSubj?: string; } -/** Exported for testing only **/ -export const filterAndConvertFields = ( - fields: FieldSpec[], - indexPattern?: string, - fieldPrefix?: string -) => { - if (!fields) return {}; - let filteredFields: FieldSpec[] = []; - - if (fieldPrefix) { - // exclude fields from different indices - if (indexPattern === INDEX_NAME) { - filteredFields = fields.filter((field) => field.name.startsWith(fieldPrefix)); +const getMappings = (indexPattern: string) => { + switch (indexPattern) { + case `.${AGENTS_PREFIX}`: + return AGENT_MAPPINGS; + case `.${AGENT_POLICY_SAVED_OBJECT_TYPE}`: + return AGENT_POLICY_MAPPINGS; + case `.${PACKAGE_POLICY_SAVED_OBJECT_TYPE}`: + return PACKAGE_POLICIES_MAPPINGS; + case `.${FLEET_ENROLLMENT_API_PREFIX}`: + return ENROLLMENT_API_KEY_MAPPINGS; + default: + return {}; + } +}; + +const getType = (type: string) => { + switch (type) { + case 'keyword': + return 'string'; + case 'text': + return 'string'; + case 'version': + return 'string'; + case 'integer': + return 'number'; + case 'double': + return 'number'; + default: + return type; + } +}; + +const concatKeys = (obj: any, parentKey = '') => { + let result: string[] = []; + for (const key in obj) { + if (typeof obj[key] === 'object') { + result = result.concat(concatKeys(obj[key], `${parentKey}${key}.`)); } else { - // filter out fields that have names to be hidden - filteredFields = fields.filter((field) => { - for (const hiddenField of HIDDEN_FIELDS) { - if (field.name.includes(hiddenField)) { - return false; - } - } - return true; - }); + result.push(`${parentKey}${key}:${obj[key]}`); } - } else { - filteredFields = fields; } - - const fieldsMap = filteredFields.reduce((acc: Record, curr: FieldSpec) => { - acc[curr.name] = curr; - return acc; - }, {}); - return fieldsMap; + return result; +}; +/** Exported for testing only **/ +export const getFieldSpecs = (indexPattern: string) => { + const mapping = getMappings(indexPattern); + // @ts-ignore-next-line + const rawFields = concatKeys(mapping?.properties) || []; + const fields = rawFields + .map((field) => field.replaceAll(/.properties/g, '')) + .map((field) => field.replace(/.type/g, '')) + .map((field) => field.split(':')); + + const fieldSpecs: FieldSpec[] = fields.map((field) => { + return { + name: field[0], + type: getType(field[1]), + searchable: true, + aggregatable: true, + esTypes: [field[1]], + }; + }); + return fieldSpecs; }; export const SearchBar: React.FunctionComponent = ({ @@ -108,16 +148,11 @@ export const SearchBar: React.FunctionComponent = ({ useEffect(() => { const fetchFields = async () => { try { - const fields: FieldSpec[] = await data.dataViews.getFieldsForWildcard({ - pattern: indexPattern, - }); - const fieldsMap = filterAndConvertFields(fields, indexPattern, fieldPrefix); - // Refetch only if fieldsMap is empty - const skipFetchField = !!fieldsMap; - + const fieldSpecs = getFieldSpecs(indexPattern); + const fieldsMap = data.dataViews.fieldArrayToMap(fieldSpecs); const newDataView = await data.dataViews.create( { title: indexPattern, fields: fieldsMap }, - skipFetchField + true ); setDataView(newDataView); } catch (err) { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.tsx index ab742ee54f0d3..e7c691fce5ea9 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.tsx @@ -260,6 +260,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => { }); setSearch(newSearch); }} + indexPattern={`.${AGENT_POLICY_SAVED_OBJECT_TYPE}`} fieldPrefix={AGENT_POLICY_SAVED_OBJECT_TYPE} dataTestSubj="agentPolicyList.queryInput" /> diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 37e570648e392..59f7ce0d95f75 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -98,4 +98,4 @@ export { } from './fleet_es_assets'; export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets'; export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets'; -export * from './mappings'; +export * from '../../common/constants/mappings'; From 4ecbb15efe3cf65e3131c737d536330e18bc51d0 Mon Sep 17 00:00:00 2001 From: criamico Date: Mon, 4 Dec 2023 12:14:54 +0100 Subject: [PATCH 2/2] Fix failing test --- .../public/applications/fleet/components/search_bar.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx index ebbbeb0814bcf..cda143a3343d6 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.test.tsx @@ -170,7 +170,7 @@ describe('SearchBar', () => { describe('getFieldSpecs', () => { it('returns fieldSpecs for fleet-agents', () => { - expect(getFieldSpecs(`.${AGENTS_PREFIX}`)).toHaveLength(66); + expect(getFieldSpecs(`.${AGENTS_PREFIX}`)).toHaveLength(57); }); it('returns getFieldSpecs for fleet-enrollment-api-keys', () => { const indexPattern = `.${FLEET_ENROLLMENT_API_PREFIX}`;