From 6fe6cdd9d59a0e284dfcacf4b982fa33c261e014 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 6 Dec 2023 10:00:47 -0500 Subject: [PATCH] [Fleet] Support Elasticsearch output performance presets (#172359) ## Summary Closes https://github.com/elastic/kibana/issues/166870 Closes https://github.com/elastic/kibana/issues/172525 - Adds a new `preset` field to output saved objects - Updates REST spec payloads to allow `preset` field in `POST/PUT` requests to the `/api/fleet/outputs` endpoint - Adds logic to set default `preset` to `balanced` or `custom` based on whether a reserved key exists in `output.config_yaml` - Adds UI to the output settings flyout for providing a preset - Adds backfill logic to Fleet `setup` that updates all existing outputs + redeploys their associated policies to ensure the proper `preset` is provided on all policies ## To do - [x] Fix failing tests - [x] Add a lot of tests + testing instructions - [x] Allow preconfigured outputs to specify a preset - [x] Update OpenAPI spec for outputs API - [x] Disable `EuiSelect` when output is managed - [x] Add in-product link to performance preset docs once they exist (might have to be a follow-up? (Follow up: https://github.com/elastic/kibana/issues/172523) - [x] Parse YML box contents instead of using basic string lookup for forcing `custom` preset (Follow up: https://github.com/elastic/kibana/issues/172525) ## How to test 1. Create a new Elasticsearch output 2. Observe the `Performance preset` dropdown defaults to `balanced` 3. Add a performance setting to the custom YAML box e.g. `bulk_max_size: 1000` 4. Note the callout with the list of reserved keys 5. Note that the dropdown switches to `Custom` and is now disabled 6. Remove the offending key 7. Note the dropdown returns to its normal state 8. Save the output 9. Edit the output and observe the same behaviors For the backfill 1. Create a local environment with multiple elasticsearch outputs on `main` 2. Stop Kibana 3. Checkout this PR branch 4. Restart Kibana 5. Observe the ES outputs have been updated to include the appropriate `preset` value ## Screenshots + Screen recordings https://github.com/elastic/kibana/assets/6766512/0c25a15e-938d-4747-8846-d51a9ad01968 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../current_fields.json | 1 + .../current_mappings.json | 4 + .../check_registered_types.test.ts | 2 +- .../plugins/fleet/common/constants/output.ts | 17 +- .../plugins/fleet/common/openapi/bundled.json | 20 +++ .../plugins/fleet/common/openapi/bundled.yaml | 16 ++ .../output_create_request_elasticsearch.yaml | 3 + .../output_update_request_elasticsearch.yaml | 3 + .../common/services/output_helpers.test.ts | 59 ++++++- .../fleet/common/services/output_helpers.ts | 43 ++++- .../fleet/common/types/models/output.ts | 4 + .../cypress/e2e/fleet_settings_outputs.cy.ts | 99 +++++++++++- x-pack/plugins/fleet/cypress/screens/fleet.ts | 1 + .../fleet/cypress/screens/fleet_outputs.ts | 12 ++ .../components/edit_output_flyout/index.tsx | 148 +++++++++++++++--- .../output_form_remote_es.tsx | 1 + .../edit_output_flyout/use_output_form.tsx | 30 +++- .../fleet/sections/settings/index.tsx | 2 +- .../plugins/fleet/server/errors/handlers.ts | 1 + .../cloud_preconfiguration.test.ts | 1 + .../fleet/server/saved_objects/index.ts | 18 +++ .../full_agent_policy.test.ts.snap | 6 + .../agent_policies/full_agent_policy.test.ts | 4 + .../agent_policies/full_agent_policy.ts | 26 ++- .../fleet/server/services/output.test.ts | 124 ++++++++++++++- .../plugins/fleet/server/services/output.ts | 88 ++++++++++- .../services/preconfiguration/outputs.ts | 2 + x-pack/plugins/fleet/server/services/setup.ts | 3 + .../fleet/server/types/models/output.ts | 18 +++ .../fleet/server/types/so_attributes.ts | 2 + .../apis/agents/upgrade.ts | 1 + .../apis/outputs/crud.ts | 107 +++++++++++++ 32 files changed, 812 insertions(+), 54 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index ad25525f19303..e2f6f916b3dda 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -528,6 +528,7 @@ "output_id", "partition", "password", + "preset", "proxy_id", "random", "random.group_events", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index fcc8422c45865..8cedbf8722772 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1854,6 +1854,10 @@ } } } + }, + "preset": { + "type": "keyword", + "index": false } } }, diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 43920d3bbbe23..f5127f499644e 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "20bd44ce6016079c3b28f1b2bc241e7715be48f8", + "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 376e2c363225f..4d75f79df0a26 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { NewOutput } from '../types'; +import type { NewOutput, OutputType, ValueOf } from '../types'; export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs'; @@ -120,4 +120,19 @@ export const kafkaSupportedVersions = [ '2.6.0', ]; +export const RESERVED_CONFIG_YML_KEYS = [ + 'bulk_max_size', + 'workers', + 'queue.mem.events', + 'flush.min_events', + 'flush.timeout', + 'compression', + 'idle_timeout', +]; + +export const OUTPUT_TYPES_WITH_PRESET_SUPPORT: Array> = [ + outputType.Elasticsearch, + outputType.RemoteElasticsearch, +]; + export const OUTPUT_HEALTH_DATA_STREAM = 'logs-fleet_server.output_health-default'; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 580191a0bff2f..e204b0fb7885a 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7952,6 +7952,16 @@ "config": { "type": "object" }, + "preset": { + "type": "string", + "enum": [ + "balanced", + "custom", + "throughput", + "scale", + "latency" + ] + }, "config_yaml": { "type": "string" }, @@ -8442,6 +8452,16 @@ "config": { "type": "object" }, + "preset": { + "type": "string", + "enum": [ + "balanced", + "custom", + "throughput", + "scale", + "latency" + ] + }, "config_yaml": { "type": "string" }, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 8bd2f5204f4f7..415089616e1a5 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -5130,6 +5130,14 @@ components: type: string config: type: object + preset: + type: string + enum: + - balanced + - custom + - throughput + - scale + - latency config_yaml: type: string ssl: @@ -5451,6 +5459,14 @@ components: type: string config: type: object + preset: + type: string + enum: + - balanced + - custom + - throughput + - scale + - latency config_yaml: type: string ssl: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml index aefa17057e065..0af1da40121d5 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml @@ -22,6 +22,9 @@ properties: type: string config: type: object + preset: + type: string + enum: ['balanced', 'custom', 'throughput', 'scale', 'latency'] config_yaml: type: string ssl: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml index bc10be5922883..570d0da0138bf 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml @@ -22,6 +22,9 @@ properties: type: string config: type: object + preset: + type: string + enum: ['balanced', 'custom', 'throughput', 'scale', 'latency'] config_yaml: type: string ssl: diff --git a/x-pack/plugins/fleet/common/services/output_helpers.test.ts b/x-pack/plugins/fleet/common/services/output_helpers.test.ts index 13928eff31ca0..1c3db129fee2f 100644 --- a/x-pack/plugins/fleet/common/services/output_helpers.test.ts +++ b/x-pack/plugins/fleet/common/services/output_helpers.test.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { getAllowedOutputTypeForPolicy } from './output_helpers'; +import { safeLoad } from 'js-yaml'; + +import { + getAllowedOutputTypeForPolicy, + outputYmlIncludesReservedPerformanceKey, +} from './output_helpers'; describe('getAllowedOutputTypeForPolicy', () => { it('should return all available output type for an agent policy without APM and Fleet Server', () => { @@ -45,3 +50,55 @@ describe('getAllowedOutputTypeForPolicy', () => { expect(res).toEqual(['elasticsearch']); }); }); + +describe('outputYmlIncludesReservedPerformanceKey', () => { + describe('dot notation', () => { + it('returns true when reserved key is present', () => { + const configYml = `queue.mem.events: 1000`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true); + }); + + it('returns false when no reserved key is present', () => { + const configYml = `some.random.key: 1000`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false); + }); + }); + + describe('object notation', () => { + it('returns true when reserved key is present', () => { + const configYml = ` + queue: + mem: + events: 1000 + `; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true); + }); + + it('returns false when no reserved key is present', () => { + const configYml = ` + some: + random: + key: 1000 + `; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false); + }); + }); + + describe('plain string', () => { + it('returns true when reserved key is present', () => { + const configYml = `bulk_max_size`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true); + }); + + it('returns false when no reserved key is present', () => { + const configYml = `just a string`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false); + }); + }); +}); diff --git a/x-pack/plugins/fleet/common/services/output_helpers.ts b/x-pack/plugins/fleet/common/services/output_helpers.ts index 00b480c378b61..0db73e4d83511 100644 --- a/x-pack/plugins/fleet/common/services/output_helpers.ts +++ b/x-pack/plugins/fleet/common/services/output_helpers.ts @@ -5,12 +5,18 @@ * 2.0. */ -import type { AgentPolicy } from '../types'; +import { isObject } from 'lodash'; + +import { getFlattenedObject } from '@kbn/std'; + +import type { AgentPolicy, OutputType, ValueOf } from '../types'; import { FLEET_APM_PACKAGE, FLEET_SERVER_PACKAGE, FLEET_SYNTHETICS_PACKAGE, outputType, + OUTPUT_TYPES_WITH_PRESET_SUPPORT, + RESERVED_CONFIG_YML_KEYS, } from '../constants'; /** @@ -33,3 +39,38 @@ export function getAllowedOutputTypeForPolicy(agentPolicy: AgentPolicy) { return Object.values(outputType); } + +export function outputYmlIncludesReservedPerformanceKey( + configYml: string, + // Dependency injection for `safeLoad` prevents bundle size issues 🤷‍♀️ + safeLoad: (yml: string) => any +) { + if (!configYml || configYml === '') { + return false; + } + + const parsedYml = safeLoad(configYml); + + if (!isObject(parsedYml)) { + return RESERVED_CONFIG_YML_KEYS.some((key) => parsedYml.includes(key)); + } + + const flattenedYml = isObject(parsedYml) ? getFlattenedObject(parsedYml) : {}; + + return RESERVED_CONFIG_YML_KEYS.some((key) => Object.keys(flattenedYml).includes(key)); +} + +export function getDefaultPresetForEsOutput( + configYaml: string, + safeLoad: (yml: string) => any +): 'balanced' | 'custom' { + if (outputYmlIncludesReservedPerformanceKey(configYaml, safeLoad)) { + return 'custom'; + } + + return 'balanced'; +} + +export function outputTypeSupportPresets(type: ValueOf) { + return OUTPUT_TYPES_WITH_PRESET_SUPPORT.includes(type); +} diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 8d721509495ea..842ade5452963 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -29,6 +29,9 @@ export type OutputSecret = id: string; hash?: string; }; + +export type OutputPreset = 'custom' | 'balanced' | 'throughput' | 'scale' | 'latency'; + interface NewBaseOutput { is_default: boolean; is_default_monitoring: boolean; @@ -49,6 +52,7 @@ interface NewBaseOutput { shipper?: ShipperOutput | null; allow_edit?: string[]; secrets?: {}; + preset?: OutputPreset; } export interface NewElasticsearchOutput extends NewBaseOutput { diff --git a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts index 3b312241fb70f..28b124116502a 100644 --- a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts @@ -22,7 +22,9 @@ import { loadKafkaOutput, loadLogstashOutput, resetKafkaOutputForm, + selectESOutput, selectKafkaOutput, + selectRemoteESOutput, shouldDisplayError, validateOutputTypeChangeToKafka, validateSavedKafkaOutputForm, @@ -37,6 +39,101 @@ describe('Outputs', () => { login(); }); + describe('Elasticsearch', () => { + describe('Preset input', () => { + it('is set to balanced by default', () => { + selectESOutput(); + + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', 'balanced'); + }); + + it('forces custom when reserved key is included in config YAML box', () => { + selectESOutput(); + + cy.getBySel('kibanaCodeEditor').click().focused().type('bulk_max_size: 1000'); + + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT) + .should('have.value', 'custom') + .should('be.disabled'); + }); + + it('allows balanced when reserved key is not included in config yaml box', () => { + selectESOutput(); + + cy.getBySel('kibanaCodeEditor').click().focused().type('some_random_key: foo'); + + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT) + .should('have.value', 'balanced') + .should('be.enabled'); + }); + + const cases = [ + { + name: 'Preset: Balanced', + preset: 'balanced', + }, + { + name: 'Preset: Custom', + preset: 'custom', + }, + { + name: 'Preset: Throughput', + preset: 'throughput', + }, + { + name: 'Preset: Scale', + preset: 'scale', + }, + { + name: 'Preset: Latency', + preset: 'latency', + }, + ]; + + for (const type of ['elasticsearch', 'remote_elasticsearch']) { + for (const testCase of cases) { + describe(`When type is ${type} and preset is ${testCase.preset}`, () => { + it('successfully creates output', () => { + if (type === 'elasticsearch') { + selectESOutput(); + } else if (type === 'remote_elasticsearch') { + selectRemoteESOutput(); + } + + cy.getBySel(SETTINGS_OUTPUTS.NAME_INPUT).type(`${type} - ${testCase.name}`); + cy.get(`[placeholder="Specify host URL"]`).type( + `http://${testCase.preset}.elasticsearch.com:9200` + ); + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).select(testCase.preset); + + if (type === 'remote_elasticsearch') { + cy.getBySel('serviceTokenSecretInput').type('secret'); + } + + cy.intercept('POST', '**/api/fleet/outputs').as('saveOutput'); + cy.getBySel(SETTINGS_SAVE_BTN).click(); + + cy.wait('@saveOutput').then((interception) => { + const responseBody = interception.response?.body; + cy.visit(`/app/fleet/settings/outputs/${responseBody?.item?.id}`); + }); + + cy.getBySel(SETTINGS_OUTPUTS.NAME_INPUT).should( + 'have.value', + `${type} - ${testCase.name}` + ); + cy.get(`[placeholder="Specify host URL"]`).should( + 'have.value', + `http://${testCase.preset}.elasticsearch.com:9200` + ); + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', testCase.preset); + }); + }); + } + } + }); + }); + describe('Kafka', () => { describe('Form validation', () => { it('renders all form fields', () => { @@ -204,7 +301,7 @@ describe('Outputs', () => { cy.contains('Name is required'); cy.contains('Host is required'); cy.contains('Username is required'); - // cy.contains('Password is required'); // TODO + // cy.contains('Password is required'); cy.contains('Default topic is required'); cy.contains('Topic is required'); cy.contains( diff --git a/x-pack/plugins/fleet/cypress/screens/fleet.ts b/x-pack/plugins/fleet/cypress/screens/fleet.ts index fa623389e7c92..1d0acb2f34739 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet.ts @@ -124,6 +124,7 @@ export const SETTINGS_OUTPUTS = { ADD_HOST_ROW_BTN: 'fleetServerHosts.multiRowInput.addRowButton', WARNING_KAFKA_CALLOUT: 'settingsOutputsFlyout.kafkaOutputTypeCallout', WARNING_ELASTICSEARCH_CALLOUT: 'settingsOutputsFlyout.elasticsearchOutputTypeCallout', + PRESET_INPUT: 'settingsOutputsFlyout.presetInput', }; export const getSpecificSelectorId = (selector: string, id: number) => { diff --git a/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts b/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts index 5b3b0f8d4af16..e1d1a3530df9a 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts @@ -15,6 +15,18 @@ import { SETTINGS_SAVE_BTN, } from './fleet'; +export const selectESOutput = () => { + visit('/app/fleet/settings'); + cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); + cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('elasticsearch'); +}; + +export const selectRemoteESOutput = () => { + visit('/app/fleet/settings'); + cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); + cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('remote_elasticsearch'); +}; + export const selectKafkaOutput = () => { visit('/app/fleet/settings'); cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx index 7ceef20111042..d9fa988aa55cc 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx @@ -7,6 +7,7 @@ import React, { useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; +import { safeLoad } from 'js-yaml'; import { EuiFlyout, @@ -31,14 +32,23 @@ import { EuiBetaBadge, useEuiTheme, EuiText, + EuiAccordion, + EuiCode, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { css } from '@emotion/react'; +import type { OutputType, ValueOf } from '../../../../../../../common/types'; + +import { + outputTypeSupportPresets, + outputYmlIncludesReservedPerformanceKey, +} from '../../../../../../../common/services/output_helpers'; + import { ExperimentalFeaturesService } from '../../../../../../services'; -import { outputType } from '../../../../../../../common/constants'; +import { outputType, RESERVED_CONFIG_YML_KEYS } from '../../../../../../../common/constants'; import { MultiRowInput } from '../multi_row_input'; import type { Output, FleetProxy } from '../../../../types'; @@ -87,7 +97,13 @@ export const EditOutputFlyout: React.FunctionComponent = const { kafkaOutput: isKafkaOutputEnabled, remoteESOutput: isRemoteESOutputEnabled } = ExperimentalFeaturesService.get(); + const isRemoteESOutput = inputs.typeInput.value === outputType.RemoteElasticsearch; + const isESOutput = inputs.typeInput.value === outputType.Elasticsearch; + const supportsPresets = inputs.typeInput.value + ? outputTypeSupportPresets(inputs.typeInput.value as ValueOf) + : false; + // Remote ES output not yet supported in serverless const isStateful = !cloud?.isServerlessEnabled; @@ -312,7 +328,6 @@ export const EditOutputFlyout: React.FunctionComponent = }; const renderTypeSpecificWarning = () => { - const isESOutput = inputs.typeInput.value === outputType.Elasticsearch; const isKafkaOutput = inputs.typeInput.value === outputType.Kafka; if (!isKafkaOutput && !isESOutput && !isRemoteESOutput) { return null; @@ -507,30 +522,6 @@ export const EditOutputFlyout: React.FunctionComponent = /> )} - - {i18n.translate('xpack.fleet.settings.editOutputFlyout.yamlConfigInputLabel', { - defaultMessage: 'Advanced YAML configuration', - })} - - } - {...inputs.additionalYamlConfigInput.formRowProps} - fullWidth - > - - = } /> + {supportsPresets && ( + <> + + + } + > + <> + inputs.presetInput.setValue(e.target.value)} + disabled={ + inputs.presetInput.props.disabled || + outputYmlIncludesReservedPerformanceKey( + inputs.additionalYamlConfigInput.value, + safeLoad + ) + } + options={[ + { value: 'balanced', text: 'Balanced' }, + { value: 'custom', text: 'Custom' }, + { value: 'throughput', text: 'Throughput' }, + { value: 'scale', text: 'Scale' }, + { value: 'latency', text: 'Latency' }, + ]} + /> + + + + )} + + {supportsPresets && + outputYmlIncludesReservedPerformanceKey( + inputs.additionalYamlConfigInput.value, + safeLoad + ) && ( + <> + + + } + > + + } + > +
    + {RESERVED_CONFIG_YML_KEYS.map((key) => ( +
  • + {key} +
    • + ))} +
+ + + + )} + + + {i18n.translate('xpack.fleet.settings.editOutputFlyout.yamlConfigInputLabel', { + defaultMessage: 'Advanced YAML configuration', + })} + + } + {...inputs.additionalYamlConfigInput.formRowProps} + fullWidth + > + { + if (outputYmlIncludesReservedPerformanceKey(value, safeLoad)) { + inputs.presetInput.setValue('custom'); + } + + inputs.additionalYamlConfigInput.setValue(value); + }} + disabled={inputs.additionalYamlConfigInput.props.disabled} + placeholder={i18n.translate( + 'xpack.fleet.settings.editOutputFlyout.yamlConfigInputPlaceholder', + { + defaultMessage: + '# YAML settings here will be added to the output section of each agent policy.', + } + )} + /> + {output?.id && output.type === 'remote_elasticsearch' ? ( diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx index 9e5fe1bc519fb..c704972a98e90 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx @@ -54,6 +54,7 @@ export const OutputFormRemoteEsSection: React.FunctionComponent = (props) > ; compressionLevelInput: ReturnType; logstashHostsInput: ReturnType; + presetInput: ReturnType; additionalYamlConfigInput: ReturnType; defaultOutputInput: ReturnType; defaultMonitoringOutputInput: ReturnType; @@ -176,6 +179,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { if (!isPreconfigured) { return false; } + return !allowEdit.includes(field); } @@ -211,6 +215,12 @@ export function useOutputForm(onSucess: () => void, output?: Output) { isDisabled('hosts') ); + const presetInput = useInput( + output?.preset ?? getDefaultPresetForEsOutput(output?.config_yaml ?? '', safeLoad), + () => undefined, + isDisabled('preset') + ); + // Remtote ES inputs const serviceTokenInput = useInput( (output as NewRemoteElasticsearchOutput)?.service_token ?? '', @@ -506,6 +516,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { diskQueueCompressionEnabled, compressionLevelInput, logstashHostsInput, + presetInput, additionalYamlConfigInput, defaultOutputInput, defaultMonitoringOutputInput, @@ -865,6 +876,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { hosts: elasticsearchUrlInput.value, is_default: false, is_default_monitoring: defaultMonitoringOutputInput.value, + preset: presetInput.value, config_yaml: additionalYamlConfigInput.value, service_token: serviceTokenInput.value || undefined, ...(!serviceTokenInput.value && @@ -884,6 +896,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { hosts: elasticsearchUrlInput.value, is_default: defaultOutputInput.value, is_default_monitoring: defaultMonitoringOutputInput.value, + preset: presetInput.value, config_yaml: additionalYamlConfigInput.value, ca_trusted_fingerprint: caTrustedFingerprintInput.value, proxy_id: proxyIdValue, @@ -940,26 +953,26 @@ export function useOutputForm(onSucess: () => void, output?: Output) { loadBalanceEnabledInput.value, typeInput.value, kafkaSslCertificateAuthoritiesInput.value, - kafkaCompressionInput.value, + kafkaSslKeyInput, + kafkaSslKeySecretInput, + kafkaAuthPasswordInput, + kafkaAuthPasswordSecretInput, nameInput.value, kafkaHostsInput.value, defaultOutputInput.value, defaultMonitoringOutputInput.value, additionalYamlConfigInput.value, + kafkaConnectionTypeInput.value, kafkaAuthMethodInput.value, kafkaSslCertificateInput.value, - kafkaSslKeyInput, - kafkaSslKeySecretInput, kafkaVerificationModeInput.value, kafkaClientIdInput.value, kafkaVersionInput.value, kafkaKeyInput.value, + kafkaCompressionInput.value, kafkaCompressionCodecInput.value, kafkaCompressionLevelInput.value, - kafkaConnectionTypeInput.value, kafkaAuthUsernameInput.value, - kafkaAuthPasswordInput, - kafkaAuthPasswordSecretInput, kafkaSaslMechanismInput.value, kafkaPartitionTypeInput.value, kafkaPartitionTypeRandomInput.value, @@ -974,12 +987,13 @@ export function useOutputForm(onSucess: () => void, output?: Output) { logstashHostsInput.value, sslCertificateInput.value, sslKeyInput.value, - sslKeySecretInput.value, sslCertificateAuthoritiesInput.value, + sslKeySecretInput.value, elasticsearchUrlInput.value, - caTrustedFingerprintInput.value, serviceTokenInput.value, serviceTokenSecretInput.value, + presetInput.value, + caTrustedFingerprintInput.value, confirm, notifications.toasts, ]); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx index 34989d2920e73..420b1ef55ee11 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx @@ -27,11 +27,11 @@ import { FleetServerFlyout } from '../../components'; import { SettingsPage } from './components/settings_page'; import { withConfirmModalProvider } from './hooks/use_confirm_modal'; import { FleetServerHostsFlyout } from './components/fleet_server_hosts_flyout'; -import { EditOutputFlyout } from './components/edit_output_flyout'; import { useDeleteOutput, useDeleteFleetServerHost, useDeleteProxy } from './hooks'; import { EditDownloadSourceFlyout } from './components/download_source_flyout'; import { useDeleteDownloadSource } from './components/download_source_flyout/use_delete_download_source'; import { FleetProxyFlyout } from './components/edit_fleet_proxy_flyout'; +import { EditOutputFlyout } from './components/edit_output_flyout'; function useSettingsAppData() { const outputs = useGetOutputs(); diff --git a/x-pack/plugins/fleet/server/errors/handlers.ts b/x-pack/plugins/fleet/server/errors/handlers.ts index 7bd380af258e2..3bfe94537c587 100644 --- a/x-pack/plugins/fleet/server/errors/handlers.ts +++ b/x-pack/plugins/fleet/server/errors/handlers.ts @@ -121,6 +121,7 @@ const getHTTPResponseCode = (error: FleetError): number => { // Connection errors (ie. RegistryConnectionError) / fallback (RegistryError) from EPR return 502; } + return 400; // Bad Request }; diff --git a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts index c080aefacd12e..72335f2c94f31 100644 --- a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts @@ -333,6 +333,7 @@ describe('Fleet cloud preconfiguration', () => { 'es-containerhost': { hosts: ['https://cloudinternales:9200'], type: 'elasticsearch', + preset: 'balanced', }, }, revision: 5, diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index b1fa2c0e462ee..ff569adfd95b5 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -278,6 +278,10 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, }, }, + preset: { + type: 'keyword', + index: false, + }, }, }, modelVersions: { @@ -329,6 +333,19 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, ], }, + '4': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + preset: { + type: 'keyword', + index: false, + }, + }, + }, + ], + }, }, migrations: { '7.13.0': migrateOutputToV7130, @@ -697,6 +714,7 @@ export function registerEncryptedSavedObjects( 'timeout', 'broker_timeout', 'required_acks', + 'preset', 'secrets', ]), }); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index df17382b1cd3f..86f8126c1c45a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -58,12 +58,14 @@ Object { "hosts": Array [ "http://es-data.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, "default": Object { "hosts": Array [ "http://127.0.0.1:9201", ], + "preset": "balanced", "type": "elasticsearch", }, }, @@ -134,12 +136,14 @@ Object { "hosts": Array [ "http://127.0.0.1:9201", ], + "preset": "balanced", "type": "elasticsearch", }, "monitoring-output-id": Object { "hosts": Array [ "http://es-monitoring.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, }, @@ -210,12 +214,14 @@ Object { "hosts": Array [ "http://es-data.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, "monitoring-output-id": Object { "hosts": Array [ "http://es-monitoring.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, }, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index d9bf271210bc9..3250c55d5f6e7 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -485,6 +485,7 @@ describe('transformOutputToFullPolicyOutput', () => { "hosts": Array [ "http://host.fr", ], + "preset": "balanced", "type": "elasticsearch", } `); @@ -509,6 +510,7 @@ ssl.test: 123 "hosts": Array [ "http://host.fr", ], + "preset": "balanced", "ssl.ca_trusted_fingerprint": "fingerprint123", "ssl.test": 123, "test": 1234, @@ -541,6 +543,7 @@ ssl.test: 123 "hosts": Array [ "http://host.fr", ], + "preset": "balanced", "proxy_url": "https://proxy1.fr", "type": "elasticsearch", } @@ -567,6 +570,7 @@ ssl.test: 123 "http://host.fr", ], "password": "\${ES_PASSWORD}", + "preset": "balanced", "type": "elasticsearch", "username": "\${ES_USERNAME}", } diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index dcb9ac0d0b89f..154614d2aa297 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -5,10 +5,17 @@ * 2.0. */ +/* eslint-disable @typescript-eslint/naming-convention */ + import type { SavedObjectsClientContract } from '@kbn/core/server'; import { safeLoad } from 'js-yaml'; import deepMerge from 'deepmerge'; +import { + getDefaultPresetForEsOutput, + outputTypeSupportPresets, +} from '../../../common/services/output_helpers'; + import type { FullAgentPolicy, PackagePolicy, @@ -311,9 +318,17 @@ export function transformOutputToFullPolicyOutput( proxy?: FleetProxy, standalone = false ): FullAgentPolicyOutput { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl, shipper, secrets } = - output; + const { + config_yaml, + type, + hosts, + ca_sha256, + ca_trusted_fingerprint, + ssl, + shipper, + secrets, + preset, + } = output; const configJs = config_yaml ? safeLoad(config_yaml) : {}; @@ -324,7 +339,6 @@ export function transformOutputToFullPolicyOutput( let kafkaData = {}; if (type === outputType.Kafka) { - /* eslint-disable @typescript-eslint/naming-convention */ const { client_id, version, @@ -482,6 +496,10 @@ export function transformOutputToFullPolicyOutput( newOutput.service_token = output.service_token; } + if (outputTypeSupportPresets(output.type)) { + newOutput.preset = preset ?? getDefaultPresetForEsOutput(config_yaml ?? '', safeLoad); + } + return newOutput; } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 6de778e9fd48a..6e7df5c7d4e50 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -11,6 +11,8 @@ import { securityMock } from '@kbn/security-plugin/server/mocks'; import type { Logger } from '@kbn/logging'; +import { RESERVED_CONFIG_YML_KEYS } from '../../common/constants'; + import type { OutputSOAttributes } from '../types'; import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; @@ -724,6 +726,122 @@ describe('Output Service', () => { `Remote elasticsearch output cannot be set as default output for integration data. Please set "is_default" to false.` ); }); + + it('should set preset: balanced by default when creating a new ES output', async () => { + const soClient = getMockedSoClient({}); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + }, + { + id: 'output-1', + } + ); + + expect(soClient.create).toBeCalledWith( + OUTPUT_SAVED_OBJECT_TYPE, + // Preset should be inferred as balanced if not provided + expect.objectContaining({ + preset: 'balanced', + }), + expect.anything() + ); + }); + + it('should set preset: custom when config_yaml contains a reserved key', async () => { + const soClient = getMockedSoClient({}); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + config_yaml: ` + bulk_max_size: 1000 + `, + }, + { + id: 'output-1', + } + ); + + expect(soClient.create).toBeCalledWith( + OUTPUT_SAVED_OBJECT_TYPE, + expect.objectContaining({ + preset: 'custom', + }), + expect.anything() + ); + }); + + it('should honor preset: custom in attributes', async () => { + const soClient = getMockedSoClient({}); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + config_yaml: ` + some_non_reserved_key: foo + `, + preset: 'custom', + }, + { + id: 'output-1', + } + ); + + expect(soClient.create).toBeCalledWith( + OUTPUT_SAVED_OBJECT_TYPE, + expect.objectContaining({ + preset: 'custom', + }), + expect.anything() + ); + }); + + it('should throw an error when preset: balanced is provided but config_yaml contains a reserved key', async () => { + const soClient = getMockedSoClient({}); + + expect( + outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + config_yaml: ` + bulk_max_size: 1000 + `, + preset: 'balanced', + }, + { + id: 'output-1', + } + ) + ).rejects.toThrow( + `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( + ', ' + )}` + ); + + expect(soClient.create).not.toBeCalled(); + }); }); describe('update', () => { @@ -931,6 +1049,7 @@ describe('Output Service', () => { type: 'elasticsearch', hosts: ['http://test:4343'], ssl: null, + preset: 'balanced', }); }); @@ -969,6 +1088,7 @@ describe('Output Service', () => { headers: null, username: null, version: null, + preset: 'balanced', }); }); @@ -1681,7 +1801,7 @@ describe('Output Service', () => { }); describe('getLatestOutputHealth', () => { - it('should return unkown state if no hits', async () => { + it('should return unknown state if no hits', async () => { esClientMock.search.mockResolvedValue({ hits: { hits: [], @@ -1691,7 +1811,7 @@ describe('Output Service', () => { const response = await outputService.getLatestOutputHealth(esClientMock, 'id'); expect(response).toEqual({ - state: 'UNKOWN', + state: 'UNKNOWN', message: '', timestamp: '', }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index d4e55ea16ef2b..32784a94f6729 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -19,6 +19,14 @@ import { SavedObjectsUtils } from '@kbn/core/server'; import _ from 'lodash'; +import pMap from 'p-map'; + +import { + getDefaultPresetForEsOutput, + outputTypeSupportPresets, + outputYmlIncludesReservedPerformanceKey, +} from '../../common/services/output_helpers'; + import type { NewOutput, Output, @@ -42,6 +50,7 @@ import { kafkaPartitionType, kafkaCompressionType, kafkaAcknowledgeReliabilityLevel, + RESERVED_CONFIG_YML_KEYS, } from '../../common/constants'; import { normalizeHostsForAgents } from '../../common/services'; import { @@ -435,6 +444,20 @@ class OutputService { ); } } + + if (outputTypeSupportPresets(data.type)) { + if ( + data.preset === 'balanced' && + outputYmlIncludesReservedPerformanceKey(output.config_yaml ?? '', safeLoad) + ) { + throw new OutputInvalidError( + `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( + ', ' + )}` + ); + } + } + const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { @@ -500,6 +523,10 @@ class OutputService { data.shipper = null; } + if (!data.preset && data.type === outputType.Elasticsearch) { + data.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', safeLoad); + } + if (output.config_yaml) { const configJs = safeLoad(output.config_yaml); const isShipperDisabled = !configJs?.shipper || configJs?.shipper?.enabled === false; @@ -752,6 +779,19 @@ class OutputService { } const updateData: Nullable> = { ...omit(data, ['ssl', 'secrets']) }; + + if (updateData.type && outputTypeSupportPresets(updateData.type)) { + if ( + updateData.preset === 'balanced' && + outputYmlIncludesReservedPerformanceKey(updateData.config_yaml ?? '', safeLoad) + ) { + throw new OutputInvalidError( + `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( + ', ' + )}` + ); + } + } if (isOutputSecretsEnabled()) { const secretsRes = await extractAndUpdateOutputSecrets({ oldOutput: originalOutput, @@ -800,6 +840,10 @@ class OutputService { // If the output type changed if (data.type && data.type !== originalOutput.type) { + if (data.type === outputType.Elasticsearch && updateData.type === outputType.Elasticsearch) { + updateData.preset = null; + } + if (data.type !== outputType.Kafka && originalOutput.type === outputType.Kafka) { removeKafkaFields(updateData as Nullable); } @@ -924,6 +968,10 @@ class OutputService { updateData.hosts = updateData.hosts.map(normalizeHostsForAgents); } + if (!data.preset && data.type === outputType.Elasticsearch) { + updateData.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', safeLoad); + } + // Remove the shipper data if the shipper is not enabled from the yaml config if (!data.config_yaml && data.shipper) { updateData.shipper = null; @@ -964,16 +1012,40 @@ class OutputService { } } + public async backfillAllOutputPresets( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient + ) { + const outputs = await this.list(soClient); + + await pMap( + outputs.items.filter((output) => outputTypeSupportPresets(output.type) && !output.preset), + async (output) => { + const preset = getDefaultPresetForEsOutput(output.config_yaml ?? '', safeLoad); + + await outputService.update(soClient, esClient, output.id, { preset }); + await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, output.id); + }, + { + concurrency: 5, + } + ); + } + async getLatestOutputHealth(esClient: ElasticsearchClient, id: string): Promise { - const response = await esClient.search({ - index: OUTPUT_HEALTH_DATA_STREAM, - query: { bool: { filter: { term: { output: id } } } }, - sort: { '@timestamp': 'desc' }, - size: 1, - }); - if (response.hits.hits.length === 0) { + const response = await esClient.search( + { + index: OUTPUT_HEALTH_DATA_STREAM, + query: { bool: { filter: { term: { output: id } } } }, + sort: { '@timestamp': 'desc' }, + size: 1, + }, + { ignore: [404] } + ); + + if (!response.hits || response.hits.hits.length === 0) { return { - state: 'UNKOWN', + state: 'UNKNOWN', message: '', timestamp: '', }; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 8b22ce1009df2..f73481ff402e3 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -385,6 +385,8 @@ async function isPreconfiguredOutputDifferentFromCurrent( isDifferent(existingOutput.config_yaml, preconfiguredOutput.config_yaml) || isDifferent(existingOutput.proxy_id, preconfiguredOutput.proxy_id) || isDifferent(existingOutput.allow_edit ?? [], preconfiguredOutput.allow_edit ?? []) || + (preconfiguredOutput.preset && + isDifferent(existingOutput.preset, preconfiguredOutput.preset)) || (await kafkaFieldsAreDifferent()) || (await logstashFieldsAreDifferent()) || (await remoteESFieldsAreDifferent()) diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index e0b3bdfea0bd3..575da165d001d 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -131,6 +131,9 @@ async function createSetupSideEffects( const defaultOutput = await outputService.ensureDefaultOutput(soClient, esClient); + logger.debug('Backfilling output performance presets'); + await outputService.backfillAllOutputPresets(soClient, esClient); + logger.debug('Setting up Fleet Elasticsearch assets'); let stepSpan = apm.startSpan('Install Fleet global assets', 'preconfiguration'); await ensureFleetGlobalEsAssets(soClient, esClient); diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index 4c391e8383a58..f9e78b6db25a0 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -115,12 +115,30 @@ export const ElasticSearchSchema = { ...BaseSchema, type: schema.literal(outputType.Elasticsearch), hosts: schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }), { minSize: 1 }), + preset: schema.maybe( + schema.oneOf([ + schema.literal('balanced'), + schema.literal('custom'), + schema.literal('throughput'), + schema.literal('scale'), + schema.literal('latency'), + ]) + ), }; const ElasticSearchUpdateSchema = { ...UpdateSchema, type: schema.maybe(schema.literal(outputType.Elasticsearch)), hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }), { minSize: 1 })), + preset: schema.maybe( + schema.oneOf([ + schema.literal('balanced'), + schema.literal('custom'), + schema.literal('throughput'), + schema.literal('scale'), + schema.literal('latency'), + ]) + ), }; /** diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index df98f0b00a424..311e5159b8f15 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -16,6 +16,7 @@ import type { KafkaAcknowledgeReliabilityLevel, KafkaConnectionTypeType, AgentUpgradeDetails, + OutputPreset, } from '../../common/types'; import type { AgentType, FleetServerAgentComponent } from '../../common/types/models'; @@ -144,6 +145,7 @@ interface OutputSoBaseAttributes { allow_edit?: string[]; output_id?: string; ssl?: string | null; // encrypted ssl field + preset?: OutputPreset; } interface OutputSoElasticsearchAttributes extends OutputSoBaseAttributes { diff --git a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts index 3e2e26e347dd8..786733b0011e2 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts @@ -9,6 +9,7 @@ import expect from '@kbn/expect'; import semver from 'semver'; import moment from 'moment'; import { AGENTS_INDEX, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common'; + import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { setupFleetAndAgents } from './services'; import { skipIfNoDockerRegistry, generateAgent, makeSnapshotVersion } from '../../helpers'; diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 67ba6b9811f38..4c580351d8b5c 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -314,6 +314,59 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }); + it('should respond 400 when setting an unknown preset', async function () { + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'some_unknown_preset', + }) + .expect(400); + }); + + it('should allow changing the preset from balanced to custom and back', async function () { + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'custom', + config_yaml: 'some_random_field: foo', + }) + .expect(200); + + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'balanced', + config_yaml: 'some_random_field: foo', + }) + .expect(200); + }); + + it('should respond 400 when changing the preset from custom to balanced with reserved key', async function () { + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'balanced', + config_yaml: 'bulk_max_size: 1000', + }) + .expect(400); + }); + it('should allow to update a non-default ES output to logstash', async function () { const { body: postResponse2 } = await supertest .post(`/api/fleet/outputs`) @@ -608,9 +661,62 @@ export default function (providerContext: FtrProviderContext) { hosts: ['https://test.fr:443'], is_default: false, is_default_monitoring: false, + preset: 'balanced', }); }); + it('should allow creating a new ES output with preset: custom', async () => { + const { body: postResponse } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + preset: 'custom', + config_yaml: 'some_random_key: foo', + }) + .expect(200); + + const { id: _, ...itemWithoutId } = postResponse.item; + expect(itemWithoutId).to.eql({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr:443'], + is_default: false, + is_default_monitoring: false, + preset: 'custom', + config_yaml: 'some_random_key: foo', + }); + }); + + it('should respond with 400 when creating a new ES output with preset: balanced and a reserved key in config_yaml', async () => { + await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + preset: 'balanced', + config_yaml: 'bulk_max_size: 1000', + }) + .expect(400); + }); + + it('should respond with 400 when creating a new ES output with an unknown preset', async () => { + await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + preset: 'some_unknown_preset', + }) + .expect(400); + }); + it('should allow to create a new default ES output ', async function () { const { body: postResponse } = await supertest .post(`/api/fleet/outputs`) @@ -630,6 +736,7 @@ export default function (providerContext: FtrProviderContext) { hosts: ['https://test.fr:443'], is_default: true, is_default_monitoring: false, + preset: 'balanced', }); });