diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index ad25525f19303..e2f6f916b3dda 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -528,6 +528,7 @@ "output_id", "partition", "password", + "preset", "proxy_id", "random", "random.group_events", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index fcc8422c45865..8cedbf8722772 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1854,6 +1854,10 @@ } } } + }, + "preset": { + "type": "keyword", + "index": false } } }, diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 43920d3bbbe23..f5127f499644e 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "20bd44ce6016079c3b28f1b2bc241e7715be48f8", + "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 376e2c363225f..4d75f79df0a26 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { NewOutput } from '../types'; +import type { NewOutput, OutputType, ValueOf } from '../types'; export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs'; @@ -120,4 +120,19 @@ export const kafkaSupportedVersions = [ '2.6.0', ]; +export const RESERVED_CONFIG_YML_KEYS = [ + 'bulk_max_size', + 'workers', + 'queue.mem.events', + 'flush.min_events', + 'flush.timeout', + 'compression', + 'idle_timeout', +]; + +export const OUTPUT_TYPES_WITH_PRESET_SUPPORT: Array> = [ + outputType.Elasticsearch, + outputType.RemoteElasticsearch, +]; + export const OUTPUT_HEALTH_DATA_STREAM = 'logs-fleet_server.output_health-default'; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 580191a0bff2f..e204b0fb7885a 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7952,6 +7952,16 @@ "config": { "type": "object" }, + "preset": { + "type": "string", + "enum": [ + "balanced", + "custom", + "throughput", + "scale", + "latency" + ] + }, "config_yaml": { "type": "string" }, @@ -8442,6 +8452,16 @@ "config": { "type": "object" }, + "preset": { + "type": "string", + "enum": [ + "balanced", + "custom", + "throughput", + "scale", + "latency" + ] + }, "config_yaml": { "type": "string" }, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 8bd2f5204f4f7..415089616e1a5 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -5130,6 +5130,14 @@ components: type: string config: type: object + preset: + type: string + enum: + - balanced + - custom + - throughput + - scale + - latency config_yaml: type: string ssl: @@ -5451,6 +5459,14 @@ components: type: string config: type: object + preset: + type: string + enum: + - balanced + - custom + - throughput + - scale + - latency config_yaml: type: string ssl: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml index aefa17057e065..0af1da40121d5 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml @@ -22,6 +22,9 @@ properties: type: string config: type: object + preset: + type: string + enum: ['balanced', 'custom', 'throughput', 'scale', 'latency'] config_yaml: type: string ssl: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml index bc10be5922883..570d0da0138bf 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml @@ -22,6 +22,9 @@ properties: type: string config: type: object + preset: + type: string + enum: ['balanced', 'custom', 'throughput', 'scale', 'latency'] config_yaml: type: string ssl: diff --git a/x-pack/plugins/fleet/common/services/output_helpers.test.ts b/x-pack/plugins/fleet/common/services/output_helpers.test.ts index 13928eff31ca0..1c3db129fee2f 100644 --- a/x-pack/plugins/fleet/common/services/output_helpers.test.ts +++ b/x-pack/plugins/fleet/common/services/output_helpers.test.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { getAllowedOutputTypeForPolicy } from './output_helpers'; +import { safeLoad } from 'js-yaml'; + +import { + getAllowedOutputTypeForPolicy, + outputYmlIncludesReservedPerformanceKey, +} from './output_helpers'; describe('getAllowedOutputTypeForPolicy', () => { it('should return all available output type for an agent policy without APM and Fleet Server', () => { @@ -45,3 +50,55 @@ describe('getAllowedOutputTypeForPolicy', () => { expect(res).toEqual(['elasticsearch']); }); }); + +describe('outputYmlIncludesReservedPerformanceKey', () => { + describe('dot notation', () => { + it('returns true when reserved key is present', () => { + const configYml = `queue.mem.events: 1000`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true); + }); + + it('returns false when no reserved key is present', () => { + const configYml = `some.random.key: 1000`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false); + }); + }); + + describe('object notation', () => { + it('returns true when reserved key is present', () => { + const configYml = ` + queue: + mem: + events: 1000 + `; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true); + }); + + it('returns false when no reserved key is present', () => { + const configYml = ` + some: + random: + key: 1000 + `; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false); + }); + }); + + describe('plain string', () => { + it('returns true when reserved key is present', () => { + const configYml = `bulk_max_size`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true); + }); + + it('returns false when no reserved key is present', () => { + const configYml = `just a string`; + + expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false); + }); + }); +}); diff --git a/x-pack/plugins/fleet/common/services/output_helpers.ts b/x-pack/plugins/fleet/common/services/output_helpers.ts index 00b480c378b61..0db73e4d83511 100644 --- a/x-pack/plugins/fleet/common/services/output_helpers.ts +++ b/x-pack/plugins/fleet/common/services/output_helpers.ts @@ -5,12 +5,18 @@ * 2.0. */ -import type { AgentPolicy } from '../types'; +import { isObject } from 'lodash'; + +import { getFlattenedObject } from '@kbn/std'; + +import type { AgentPolicy, OutputType, ValueOf } from '../types'; import { FLEET_APM_PACKAGE, FLEET_SERVER_PACKAGE, FLEET_SYNTHETICS_PACKAGE, outputType, + OUTPUT_TYPES_WITH_PRESET_SUPPORT, + RESERVED_CONFIG_YML_KEYS, } from '../constants'; /** @@ -33,3 +39,38 @@ export function getAllowedOutputTypeForPolicy(agentPolicy: AgentPolicy) { return Object.values(outputType); } + +export function outputYmlIncludesReservedPerformanceKey( + configYml: string, + // Dependency injection for `safeLoad` prevents bundle size issues 🤷‍♀️ + safeLoad: (yml: string) => any +) { + if (!configYml || configYml === '') { + return false; + } + + const parsedYml = safeLoad(configYml); + + if (!isObject(parsedYml)) { + return RESERVED_CONFIG_YML_KEYS.some((key) => parsedYml.includes(key)); + } + + const flattenedYml = isObject(parsedYml) ? getFlattenedObject(parsedYml) : {}; + + return RESERVED_CONFIG_YML_KEYS.some((key) => Object.keys(flattenedYml).includes(key)); +} + +export function getDefaultPresetForEsOutput( + configYaml: string, + safeLoad: (yml: string) => any +): 'balanced' | 'custom' { + if (outputYmlIncludesReservedPerformanceKey(configYaml, safeLoad)) { + return 'custom'; + } + + return 'balanced'; +} + +export function outputTypeSupportPresets(type: ValueOf) { + return OUTPUT_TYPES_WITH_PRESET_SUPPORT.includes(type); +} diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 8d721509495ea..842ade5452963 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -29,6 +29,9 @@ export type OutputSecret = id: string; hash?: string; }; + +export type OutputPreset = 'custom' | 'balanced' | 'throughput' | 'scale' | 'latency'; + interface NewBaseOutput { is_default: boolean; is_default_monitoring: boolean; @@ -49,6 +52,7 @@ interface NewBaseOutput { shipper?: ShipperOutput | null; allow_edit?: string[]; secrets?: {}; + preset?: OutputPreset; } export interface NewElasticsearchOutput extends NewBaseOutput { diff --git a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts index 3b312241fb70f..28b124116502a 100644 --- a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts @@ -22,7 +22,9 @@ import { loadKafkaOutput, loadLogstashOutput, resetKafkaOutputForm, + selectESOutput, selectKafkaOutput, + selectRemoteESOutput, shouldDisplayError, validateOutputTypeChangeToKafka, validateSavedKafkaOutputForm, @@ -37,6 +39,101 @@ describe('Outputs', () => { login(); }); + describe('Elasticsearch', () => { + describe('Preset input', () => { + it('is set to balanced by default', () => { + selectESOutput(); + + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', 'balanced'); + }); + + it('forces custom when reserved key is included in config YAML box', () => { + selectESOutput(); + + cy.getBySel('kibanaCodeEditor').click().focused().type('bulk_max_size: 1000'); + + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT) + .should('have.value', 'custom') + .should('be.disabled'); + }); + + it('allows balanced when reserved key is not included in config yaml box', () => { + selectESOutput(); + + cy.getBySel('kibanaCodeEditor').click().focused().type('some_random_key: foo'); + + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT) + .should('have.value', 'balanced') + .should('be.enabled'); + }); + + const cases = [ + { + name: 'Preset: Balanced', + preset: 'balanced', + }, + { + name: 'Preset: Custom', + preset: 'custom', + }, + { + name: 'Preset: Throughput', + preset: 'throughput', + }, + { + name: 'Preset: Scale', + preset: 'scale', + }, + { + name: 'Preset: Latency', + preset: 'latency', + }, + ]; + + for (const type of ['elasticsearch', 'remote_elasticsearch']) { + for (const testCase of cases) { + describe(`When type is ${type} and preset is ${testCase.preset}`, () => { + it('successfully creates output', () => { + if (type === 'elasticsearch') { + selectESOutput(); + } else if (type === 'remote_elasticsearch') { + selectRemoteESOutput(); + } + + cy.getBySel(SETTINGS_OUTPUTS.NAME_INPUT).type(`${type} - ${testCase.name}`); + cy.get(`[placeholder="Specify host URL"]`).type( + `http://${testCase.preset}.elasticsearch.com:9200` + ); + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).select(testCase.preset); + + if (type === 'remote_elasticsearch') { + cy.getBySel('serviceTokenSecretInput').type('secret'); + } + + cy.intercept('POST', '**/api/fleet/outputs').as('saveOutput'); + cy.getBySel(SETTINGS_SAVE_BTN).click(); + + cy.wait('@saveOutput').then((interception) => { + const responseBody = interception.response?.body; + cy.visit(`/app/fleet/settings/outputs/${responseBody?.item?.id}`); + }); + + cy.getBySel(SETTINGS_OUTPUTS.NAME_INPUT).should( + 'have.value', + `${type} - ${testCase.name}` + ); + cy.get(`[placeholder="Specify host URL"]`).should( + 'have.value', + `http://${testCase.preset}.elasticsearch.com:9200` + ); + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', testCase.preset); + }); + }); + } + } + }); + }); + describe('Kafka', () => { describe('Form validation', () => { it('renders all form fields', () => { @@ -204,7 +301,7 @@ describe('Outputs', () => { cy.contains('Name is required'); cy.contains('Host is required'); cy.contains('Username is required'); - // cy.contains('Password is required'); // TODO + // cy.contains('Password is required'); cy.contains('Default topic is required'); cy.contains('Topic is required'); cy.contains( diff --git a/x-pack/plugins/fleet/cypress/screens/fleet.ts b/x-pack/plugins/fleet/cypress/screens/fleet.ts index fa623389e7c92..1d0acb2f34739 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet.ts @@ -124,6 +124,7 @@ export const SETTINGS_OUTPUTS = { ADD_HOST_ROW_BTN: 'fleetServerHosts.multiRowInput.addRowButton', WARNING_KAFKA_CALLOUT: 'settingsOutputsFlyout.kafkaOutputTypeCallout', WARNING_ELASTICSEARCH_CALLOUT: 'settingsOutputsFlyout.elasticsearchOutputTypeCallout', + PRESET_INPUT: 'settingsOutputsFlyout.presetInput', }; export const getSpecificSelectorId = (selector: string, id: number) => { diff --git a/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts b/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts index 5b3b0f8d4af16..e1d1a3530df9a 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts @@ -15,6 +15,18 @@ import { SETTINGS_SAVE_BTN, } from './fleet'; +export const selectESOutput = () => { + visit('/app/fleet/settings'); + cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); + cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('elasticsearch'); +}; + +export const selectRemoteESOutput = () => { + visit('/app/fleet/settings'); + cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); + cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('remote_elasticsearch'); +}; + export const selectKafkaOutput = () => { visit('/app/fleet/settings'); cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx index 7ceef20111042..d9fa988aa55cc 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx @@ -7,6 +7,7 @@ import React, { useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; +import { safeLoad } from 'js-yaml'; import { EuiFlyout, @@ -31,14 +32,23 @@ import { EuiBetaBadge, useEuiTheme, EuiText, + EuiAccordion, + EuiCode, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { css } from '@emotion/react'; +import type { OutputType, ValueOf } from '../../../../../../../common/types'; + +import { + outputTypeSupportPresets, + outputYmlIncludesReservedPerformanceKey, +} from '../../../../../../../common/services/output_helpers'; + import { ExperimentalFeaturesService } from '../../../../../../services'; -import { outputType } from '../../../../../../../common/constants'; +import { outputType, RESERVED_CONFIG_YML_KEYS } from '../../../../../../../common/constants'; import { MultiRowInput } from '../multi_row_input'; import type { Output, FleetProxy } from '../../../../types'; @@ -87,7 +97,13 @@ export const EditOutputFlyout: React.FunctionComponent = const { kafkaOutput: isKafkaOutputEnabled, remoteESOutput: isRemoteESOutputEnabled } = ExperimentalFeaturesService.get(); + const isRemoteESOutput = inputs.typeInput.value === outputType.RemoteElasticsearch; + const isESOutput = inputs.typeInput.value === outputType.Elasticsearch; + const supportsPresets = inputs.typeInput.value + ? outputTypeSupportPresets(inputs.typeInput.value as ValueOf) + : false; + // Remote ES output not yet supported in serverless const isStateful = !cloud?.isServerlessEnabled; @@ -312,7 +328,6 @@ export const EditOutputFlyout: React.FunctionComponent = }; const renderTypeSpecificWarning = () => { - const isESOutput = inputs.typeInput.value === outputType.Elasticsearch; const isKafkaOutput = inputs.typeInput.value === outputType.Kafka; if (!isKafkaOutput && !isESOutput && !isRemoteESOutput) { return null; @@ -507,30 +522,6 @@ export const EditOutputFlyout: React.FunctionComponent = /> )} - - {i18n.translate('xpack.fleet.settings.editOutputFlyout.yamlConfigInputLabel', { - defaultMessage: 'Advanced YAML configuration', - })} - - } - {...inputs.additionalYamlConfigInput.formRowProps} - fullWidth - > - - = } /> + {supportsPresets && ( + <> + + + } + > + <> + inputs.presetInput.setValue(e.target.value)} + disabled={ + inputs.presetInput.props.disabled || + outputYmlIncludesReservedPerformanceKey( + inputs.additionalYamlConfigInput.value, + safeLoad + ) + } + options={[ + { value: 'balanced', text: 'Balanced' }, + { value: 'custom', text: 'Custom' }, + { value: 'throughput', text: 'Throughput' }, + { value: 'scale', text: 'Scale' }, + { value: 'latency', text: 'Latency' }, + ]} + /> + + + + )} + + {supportsPresets && + outputYmlIncludesReservedPerformanceKey( + inputs.additionalYamlConfigInput.value, + safeLoad + ) && ( + <> + + + } + > + + } + > +
    + {RESERVED_CONFIG_YML_KEYS.map((key) => ( +
  • + {key} +
    • + ))} +
+ + + + )} + + + {i18n.translate('xpack.fleet.settings.editOutputFlyout.yamlConfigInputLabel', { + defaultMessage: 'Advanced YAML configuration', + })} + + } + {...inputs.additionalYamlConfigInput.formRowProps} + fullWidth + > + { + if (outputYmlIncludesReservedPerformanceKey(value, safeLoad)) { + inputs.presetInput.setValue('custom'); + } + + inputs.additionalYamlConfigInput.setValue(value); + }} + disabled={inputs.additionalYamlConfigInput.props.disabled} + placeholder={i18n.translate( + 'xpack.fleet.settings.editOutputFlyout.yamlConfigInputPlaceholder', + { + defaultMessage: + '# YAML settings here will be added to the output section of each agent policy.', + } + )} + /> + {output?.id && output.type === 'remote_elasticsearch' ? ( diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx index 9e5fe1bc519fb..c704972a98e90 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx @@ -54,6 +54,7 @@ export const OutputFormRemoteEsSection: React.FunctionComponent = (props) > ; compressionLevelInput: ReturnType; logstashHostsInput: ReturnType; + presetInput: ReturnType; additionalYamlConfigInput: ReturnType; defaultOutputInput: ReturnType; defaultMonitoringOutputInput: ReturnType; @@ -176,6 +179,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { if (!isPreconfigured) { return false; } + return !allowEdit.includes(field); } @@ -211,6 +215,12 @@ export function useOutputForm(onSucess: () => void, output?: Output) { isDisabled('hosts') ); + const presetInput = useInput( + output?.preset ?? getDefaultPresetForEsOutput(output?.config_yaml ?? '', safeLoad), + () => undefined, + isDisabled('preset') + ); + // Remtote ES inputs const serviceTokenInput = useInput( (output as NewRemoteElasticsearchOutput)?.service_token ?? '', @@ -506,6 +516,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { diskQueueCompressionEnabled, compressionLevelInput, logstashHostsInput, + presetInput, additionalYamlConfigInput, defaultOutputInput, defaultMonitoringOutputInput, @@ -865,6 +876,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { hosts: elasticsearchUrlInput.value, is_default: false, is_default_monitoring: defaultMonitoringOutputInput.value, + preset: presetInput.value, config_yaml: additionalYamlConfigInput.value, service_token: serviceTokenInput.value || undefined, ...(!serviceTokenInput.value && @@ -884,6 +896,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { hosts: elasticsearchUrlInput.value, is_default: defaultOutputInput.value, is_default_monitoring: defaultMonitoringOutputInput.value, + preset: presetInput.value, config_yaml: additionalYamlConfigInput.value, ca_trusted_fingerprint: caTrustedFingerprintInput.value, proxy_id: proxyIdValue, @@ -940,26 +953,26 @@ export function useOutputForm(onSucess: () => void, output?: Output) { loadBalanceEnabledInput.value, typeInput.value, kafkaSslCertificateAuthoritiesInput.value, - kafkaCompressionInput.value, + kafkaSslKeyInput, + kafkaSslKeySecretInput, + kafkaAuthPasswordInput, + kafkaAuthPasswordSecretInput, nameInput.value, kafkaHostsInput.value, defaultOutputInput.value, defaultMonitoringOutputInput.value, additionalYamlConfigInput.value, + kafkaConnectionTypeInput.value, kafkaAuthMethodInput.value, kafkaSslCertificateInput.value, - kafkaSslKeyInput, - kafkaSslKeySecretInput, kafkaVerificationModeInput.value, kafkaClientIdInput.value, kafkaVersionInput.value, kafkaKeyInput.value, + kafkaCompressionInput.value, kafkaCompressionCodecInput.value, kafkaCompressionLevelInput.value, - kafkaConnectionTypeInput.value, kafkaAuthUsernameInput.value, - kafkaAuthPasswordInput, - kafkaAuthPasswordSecretInput, kafkaSaslMechanismInput.value, kafkaPartitionTypeInput.value, kafkaPartitionTypeRandomInput.value, @@ -974,12 +987,13 @@ export function useOutputForm(onSucess: () => void, output?: Output) { logstashHostsInput.value, sslCertificateInput.value, sslKeyInput.value, - sslKeySecretInput.value, sslCertificateAuthoritiesInput.value, + sslKeySecretInput.value, elasticsearchUrlInput.value, - caTrustedFingerprintInput.value, serviceTokenInput.value, serviceTokenSecretInput.value, + presetInput.value, + caTrustedFingerprintInput.value, confirm, notifications.toasts, ]); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx index 34989d2920e73..420b1ef55ee11 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx @@ -27,11 +27,11 @@ import { FleetServerFlyout } from '../../components'; import { SettingsPage } from './components/settings_page'; import { withConfirmModalProvider } from './hooks/use_confirm_modal'; import { FleetServerHostsFlyout } from './components/fleet_server_hosts_flyout'; -import { EditOutputFlyout } from './components/edit_output_flyout'; import { useDeleteOutput, useDeleteFleetServerHost, useDeleteProxy } from './hooks'; import { EditDownloadSourceFlyout } from './components/download_source_flyout'; import { useDeleteDownloadSource } from './components/download_source_flyout/use_delete_download_source'; import { FleetProxyFlyout } from './components/edit_fleet_proxy_flyout'; +import { EditOutputFlyout } from './components/edit_output_flyout'; function useSettingsAppData() { const outputs = useGetOutputs(); diff --git a/x-pack/plugins/fleet/server/errors/handlers.ts b/x-pack/plugins/fleet/server/errors/handlers.ts index 7bd380af258e2..3bfe94537c587 100644 --- a/x-pack/plugins/fleet/server/errors/handlers.ts +++ b/x-pack/plugins/fleet/server/errors/handlers.ts @@ -121,6 +121,7 @@ const getHTTPResponseCode = (error: FleetError): number => { // Connection errors (ie. RegistryConnectionError) / fallback (RegistryError) from EPR return 502; } + return 400; // Bad Request }; diff --git a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts index c080aefacd12e..72335f2c94f31 100644 --- a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts @@ -333,6 +333,7 @@ describe('Fleet cloud preconfiguration', () => { 'es-containerhost': { hosts: ['https://cloudinternales:9200'], type: 'elasticsearch', + preset: 'balanced', }, }, revision: 5, diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index b1fa2c0e462ee..ff569adfd95b5 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -278,6 +278,10 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, }, }, + preset: { + type: 'keyword', + index: false, + }, }, }, modelVersions: { @@ -329,6 +333,19 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, ], }, + '4': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + preset: { + type: 'keyword', + index: false, + }, + }, + }, + ], + }, }, migrations: { '7.13.0': migrateOutputToV7130, @@ -697,6 +714,7 @@ export function registerEncryptedSavedObjects( 'timeout', 'broker_timeout', 'required_acks', + 'preset', 'secrets', ]), }); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index df17382b1cd3f..86f8126c1c45a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -58,12 +58,14 @@ Object { "hosts": Array [ "http://es-data.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, "default": Object { "hosts": Array [ "http://127.0.0.1:9201", ], + "preset": "balanced", "type": "elasticsearch", }, }, @@ -134,12 +136,14 @@ Object { "hosts": Array [ "http://127.0.0.1:9201", ], + "preset": "balanced", "type": "elasticsearch", }, "monitoring-output-id": Object { "hosts": Array [ "http://es-monitoring.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, }, @@ -210,12 +214,14 @@ Object { "hosts": Array [ "http://es-data.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, "monitoring-output-id": Object { "hosts": Array [ "http://es-monitoring.co:9201", ], + "preset": "balanced", "type": "elasticsearch", }, }, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index d9bf271210bc9..3250c55d5f6e7 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -485,6 +485,7 @@ describe('transformOutputToFullPolicyOutput', () => { "hosts": Array [ "http://host.fr", ], + "preset": "balanced", "type": "elasticsearch", } `); @@ -509,6 +510,7 @@ ssl.test: 123 "hosts": Array [ "http://host.fr", ], + "preset": "balanced", "ssl.ca_trusted_fingerprint": "fingerprint123", "ssl.test": 123, "test": 1234, @@ -541,6 +543,7 @@ ssl.test: 123 "hosts": Array [ "http://host.fr", ], + "preset": "balanced", "proxy_url": "https://proxy1.fr", "type": "elasticsearch", } @@ -567,6 +570,7 @@ ssl.test: 123 "http://host.fr", ], "password": "\${ES_PASSWORD}", + "preset": "balanced", "type": "elasticsearch", "username": "\${ES_USERNAME}", } diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index dcb9ac0d0b89f..154614d2aa297 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -5,10 +5,17 @@ * 2.0. */ +/* eslint-disable @typescript-eslint/naming-convention */ + import type { SavedObjectsClientContract } from '@kbn/core/server'; import { safeLoad } from 'js-yaml'; import deepMerge from 'deepmerge'; +import { + getDefaultPresetForEsOutput, + outputTypeSupportPresets, +} from '../../../common/services/output_helpers'; + import type { FullAgentPolicy, PackagePolicy, @@ -311,9 +318,17 @@ export function transformOutputToFullPolicyOutput( proxy?: FleetProxy, standalone = false ): FullAgentPolicyOutput { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl, shipper, secrets } = - output; + const { + config_yaml, + type, + hosts, + ca_sha256, + ca_trusted_fingerprint, + ssl, + shipper, + secrets, + preset, + } = output; const configJs = config_yaml ? safeLoad(config_yaml) : {}; @@ -324,7 +339,6 @@ export function transformOutputToFullPolicyOutput( let kafkaData = {}; if (type === outputType.Kafka) { - /* eslint-disable @typescript-eslint/naming-convention */ const { client_id, version, @@ -482,6 +496,10 @@ export function transformOutputToFullPolicyOutput( newOutput.service_token = output.service_token; } + if (outputTypeSupportPresets(output.type)) { + newOutput.preset = preset ?? getDefaultPresetForEsOutput(config_yaml ?? '', safeLoad); + } + return newOutput; } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 6de778e9fd48a..6e7df5c7d4e50 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -11,6 +11,8 @@ import { securityMock } from '@kbn/security-plugin/server/mocks'; import type { Logger } from '@kbn/logging'; +import { RESERVED_CONFIG_YML_KEYS } from '../../common/constants'; + import type { OutputSOAttributes } from '../types'; import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; @@ -724,6 +726,122 @@ describe('Output Service', () => { `Remote elasticsearch output cannot be set as default output for integration data. Please set "is_default" to false.` ); }); + + it('should set preset: balanced by default when creating a new ES output', async () => { + const soClient = getMockedSoClient({}); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + }, + { + id: 'output-1', + } + ); + + expect(soClient.create).toBeCalledWith( + OUTPUT_SAVED_OBJECT_TYPE, + // Preset should be inferred as balanced if not provided + expect.objectContaining({ + preset: 'balanced', + }), + expect.anything() + ); + }); + + it('should set preset: custom when config_yaml contains a reserved key', async () => { + const soClient = getMockedSoClient({}); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + config_yaml: ` + bulk_max_size: 1000 + `, + }, + { + id: 'output-1', + } + ); + + expect(soClient.create).toBeCalledWith( + OUTPUT_SAVED_OBJECT_TYPE, + expect.objectContaining({ + preset: 'custom', + }), + expect.anything() + ); + }); + + it('should honor preset: custom in attributes', async () => { + const soClient = getMockedSoClient({}); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + config_yaml: ` + some_non_reserved_key: foo + `, + preset: 'custom', + }, + { + id: 'output-1', + } + ); + + expect(soClient.create).toBeCalledWith( + OUTPUT_SAVED_OBJECT_TYPE, + expect.objectContaining({ + preset: 'custom', + }), + expect.anything() + ); + }); + + it('should throw an error when preset: balanced is provided but config_yaml contains a reserved key', async () => { + const soClient = getMockedSoClient({}); + + expect( + outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'elasticsearch', + config_yaml: ` + bulk_max_size: 1000 + `, + preset: 'balanced', + }, + { + id: 'output-1', + } + ) + ).rejects.toThrow( + `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( + ', ' + )}` + ); + + expect(soClient.create).not.toBeCalled(); + }); }); describe('update', () => { @@ -931,6 +1049,7 @@ describe('Output Service', () => { type: 'elasticsearch', hosts: ['http://test:4343'], ssl: null, + preset: 'balanced', }); }); @@ -969,6 +1088,7 @@ describe('Output Service', () => { headers: null, username: null, version: null, + preset: 'balanced', }); }); @@ -1681,7 +1801,7 @@ describe('Output Service', () => { }); describe('getLatestOutputHealth', () => { - it('should return unkown state if no hits', async () => { + it('should return unknown state if no hits', async () => { esClientMock.search.mockResolvedValue({ hits: { hits: [], @@ -1691,7 +1811,7 @@ describe('Output Service', () => { const response = await outputService.getLatestOutputHealth(esClientMock, 'id'); expect(response).toEqual({ - state: 'UNKOWN', + state: 'UNKNOWN', message: '', timestamp: '', }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index d4e55ea16ef2b..32784a94f6729 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -19,6 +19,14 @@ import { SavedObjectsUtils } from '@kbn/core/server'; import _ from 'lodash'; +import pMap from 'p-map'; + +import { + getDefaultPresetForEsOutput, + outputTypeSupportPresets, + outputYmlIncludesReservedPerformanceKey, +} from '../../common/services/output_helpers'; + import type { NewOutput, Output, @@ -42,6 +50,7 @@ import { kafkaPartitionType, kafkaCompressionType, kafkaAcknowledgeReliabilityLevel, + RESERVED_CONFIG_YML_KEYS, } from '../../common/constants'; import { normalizeHostsForAgents } from '../../common/services'; import { @@ -435,6 +444,20 @@ class OutputService { ); } } + + if (outputTypeSupportPresets(data.type)) { + if ( + data.preset === 'balanced' && + outputYmlIncludesReservedPerformanceKey(output.config_yaml ?? '', safeLoad) + ) { + throw new OutputInvalidError( + `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( + ', ' + )}` + ); + } + } + const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { @@ -500,6 +523,10 @@ class OutputService { data.shipper = null; } + if (!data.preset && data.type === outputType.Elasticsearch) { + data.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', safeLoad); + } + if (output.config_yaml) { const configJs = safeLoad(output.config_yaml); const isShipperDisabled = !configJs?.shipper || configJs?.shipper?.enabled === false; @@ -752,6 +779,19 @@ class OutputService { } const updateData: Nullable> = { ...omit(data, ['ssl', 'secrets']) }; + + if (updateData.type && outputTypeSupportPresets(updateData.type)) { + if ( + updateData.preset === 'balanced' && + outputYmlIncludesReservedPerformanceKey(updateData.config_yaml ?? '', safeLoad) + ) { + throw new OutputInvalidError( + `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( + ', ' + )}` + ); + } + } if (isOutputSecretsEnabled()) { const secretsRes = await extractAndUpdateOutputSecrets({ oldOutput: originalOutput, @@ -800,6 +840,10 @@ class OutputService { // If the output type changed if (data.type && data.type !== originalOutput.type) { + if (data.type === outputType.Elasticsearch && updateData.type === outputType.Elasticsearch) { + updateData.preset = null; + } + if (data.type !== outputType.Kafka && originalOutput.type === outputType.Kafka) { removeKafkaFields(updateData as Nullable); } @@ -924,6 +968,10 @@ class OutputService { updateData.hosts = updateData.hosts.map(normalizeHostsForAgents); } + if (!data.preset && data.type === outputType.Elasticsearch) { + updateData.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', safeLoad); + } + // Remove the shipper data if the shipper is not enabled from the yaml config if (!data.config_yaml && data.shipper) { updateData.shipper = null; @@ -964,16 +1012,40 @@ class OutputService { } } + public async backfillAllOutputPresets( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient + ) { + const outputs = await this.list(soClient); + + await pMap( + outputs.items.filter((output) => outputTypeSupportPresets(output.type) && !output.preset), + async (output) => { + const preset = getDefaultPresetForEsOutput(output.config_yaml ?? '', safeLoad); + + await outputService.update(soClient, esClient, output.id, { preset }); + await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, output.id); + }, + { + concurrency: 5, + } + ); + } + async getLatestOutputHealth(esClient: ElasticsearchClient, id: string): Promise { - const response = await esClient.search({ - index: OUTPUT_HEALTH_DATA_STREAM, - query: { bool: { filter: { term: { output: id } } } }, - sort: { '@timestamp': 'desc' }, - size: 1, - }); - if (response.hits.hits.length === 0) { + const response = await esClient.search( + { + index: OUTPUT_HEALTH_DATA_STREAM, + query: { bool: { filter: { term: { output: id } } } }, + sort: { '@timestamp': 'desc' }, + size: 1, + }, + { ignore: [404] } + ); + + if (!response.hits || response.hits.hits.length === 0) { return { - state: 'UNKOWN', + state: 'UNKNOWN', message: '', timestamp: '', }; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 8b22ce1009df2..f73481ff402e3 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -385,6 +385,8 @@ async function isPreconfiguredOutputDifferentFromCurrent( isDifferent(existingOutput.config_yaml, preconfiguredOutput.config_yaml) || isDifferent(existingOutput.proxy_id, preconfiguredOutput.proxy_id) || isDifferent(existingOutput.allow_edit ?? [], preconfiguredOutput.allow_edit ?? []) || + (preconfiguredOutput.preset && + isDifferent(existingOutput.preset, preconfiguredOutput.preset)) || (await kafkaFieldsAreDifferent()) || (await logstashFieldsAreDifferent()) || (await remoteESFieldsAreDifferent()) diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index e0b3bdfea0bd3..575da165d001d 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -131,6 +131,9 @@ async function createSetupSideEffects( const defaultOutput = await outputService.ensureDefaultOutput(soClient, esClient); + logger.debug('Backfilling output performance presets'); + await outputService.backfillAllOutputPresets(soClient, esClient); + logger.debug('Setting up Fleet Elasticsearch assets'); let stepSpan = apm.startSpan('Install Fleet global assets', 'preconfiguration'); await ensureFleetGlobalEsAssets(soClient, esClient); diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index 4c391e8383a58..f9e78b6db25a0 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -115,12 +115,30 @@ export const ElasticSearchSchema = { ...BaseSchema, type: schema.literal(outputType.Elasticsearch), hosts: schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }), { minSize: 1 }), + preset: schema.maybe( + schema.oneOf([ + schema.literal('balanced'), + schema.literal('custom'), + schema.literal('throughput'), + schema.literal('scale'), + schema.literal('latency'), + ]) + ), }; const ElasticSearchUpdateSchema = { ...UpdateSchema, type: schema.maybe(schema.literal(outputType.Elasticsearch)), hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }), { minSize: 1 })), + preset: schema.maybe( + schema.oneOf([ + schema.literal('balanced'), + schema.literal('custom'), + schema.literal('throughput'), + schema.literal('scale'), + schema.literal('latency'), + ]) + ), }; /** diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index df98f0b00a424..311e5159b8f15 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -16,6 +16,7 @@ import type { KafkaAcknowledgeReliabilityLevel, KafkaConnectionTypeType, AgentUpgradeDetails, + OutputPreset, } from '../../common/types'; import type { AgentType, FleetServerAgentComponent } from '../../common/types/models'; @@ -144,6 +145,7 @@ interface OutputSoBaseAttributes { allow_edit?: string[]; output_id?: string; ssl?: string | null; // encrypted ssl field + preset?: OutputPreset; } interface OutputSoElasticsearchAttributes extends OutputSoBaseAttributes { diff --git a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts index 3e2e26e347dd8..786733b0011e2 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts @@ -9,6 +9,7 @@ import expect from '@kbn/expect'; import semver from 'semver'; import moment from 'moment'; import { AGENTS_INDEX, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common'; + import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { setupFleetAndAgents } from './services'; import { skipIfNoDockerRegistry, generateAgent, makeSnapshotVersion } from '../../helpers'; diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 67ba6b9811f38..4c580351d8b5c 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -314,6 +314,59 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }); + it('should respond 400 when setting an unknown preset', async function () { + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'some_unknown_preset', + }) + .expect(400); + }); + + it('should allow changing the preset from balanced to custom and back', async function () { + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'custom', + config_yaml: 'some_random_field: foo', + }) + .expect(200); + + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'balanced', + config_yaml: 'some_random_field: foo', + }) + .expect(200); + }); + + it('should respond 400 when changing the preset from custom to balanced with reserved key', async function () { + await supertest + .put(`/api/fleet/outputs/${ESOutputId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Default ES Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + preset: 'balanced', + config_yaml: 'bulk_max_size: 1000', + }) + .expect(400); + }); + it('should allow to update a non-default ES output to logstash', async function () { const { body: postResponse2 } = await supertest .post(`/api/fleet/outputs`) @@ -608,9 +661,62 @@ export default function (providerContext: FtrProviderContext) { hosts: ['https://test.fr:443'], is_default: false, is_default_monitoring: false, + preset: 'balanced', }); }); + it('should allow creating a new ES output with preset: custom', async () => { + const { body: postResponse } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + preset: 'custom', + config_yaml: 'some_random_key: foo', + }) + .expect(200); + + const { id: _, ...itemWithoutId } = postResponse.item; + expect(itemWithoutId).to.eql({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr:443'], + is_default: false, + is_default_monitoring: false, + preset: 'custom', + config_yaml: 'some_random_key: foo', + }); + }); + + it('should respond with 400 when creating a new ES output with preset: balanced and a reserved key in config_yaml', async () => { + await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + preset: 'balanced', + config_yaml: 'bulk_max_size: 1000', + }) + .expect(400); + }); + + it('should respond with 400 when creating a new ES output with an unknown preset', async () => { + await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'My output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + preset: 'some_unknown_preset', + }) + .expect(400); + }); + it('should allow to create a new default ES output ', async function () { const { body: postResponse } = await supertest .post(`/api/fleet/outputs`) @@ -630,6 +736,7 @@ export default function (providerContext: FtrProviderContext) { hosts: ['https://test.fr:443'], is_default: true, is_default_monitoring: false, + preset: 'balanced', }); });