[Fleet] Support Elasticsearch output performance presets

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -528,6 +528,7 @@
     "output_id",
     "partition",
     "password",
+    "preset",
     "proxy_id",
     "random",
     "random.group_events",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1854,6 +1854,10 @@
             }
           }
         }
+      },
+      "preset": {
+        "type": "keyword",
+        "index": false
       }
     }
   },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
-        "ingest-outputs": "20bd44ce6016079c3b28f1b2bc241e7715be48f8",
+        "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b",
         "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43",
         "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",

x-pack/plugins/fleet/common/constants/output.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { NewOutput } from '../types';
+import type { NewOutput, OutputType, ValueOf } from '../types';
 
 export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs';
 
@@ -120,4 +120,19 @@ export const kafkaSupportedVersions = [
   '2.6.0',
 ];
 
+export const RESERVED_CONFIG_YML_KEYS = [
+  'bulk_max_size',
+  'workers',
+  'queue.mem.events',
+  'flush.min_events',
+  'flush.timeout',
+  'compression',
+  'idle_timeout',
+];
+
+export const OUTPUT_TYPES_WITH_PRESET_SUPPORT: Array<ValueOf<OutputType>> = [
+  outputType.Elasticsearch,
+  outputType.RemoteElasticsearch,
+];
+
 export const OUTPUT_HEALTH_DATA_STREAM = 'logs-fleet_server.output_health-default';

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -7952,6 +7952,13 @@
           "config": {
             "type": "object"
           },
+          "preset": {
+            "type": "string",
+            "enum": [
+              "balanced",
+              "custom"
+            ]
+          },
           "config_yaml": {
             "type": "string"
           },
@@ -8442,6 +8449,13 @@
           "config": {
             "type": "object"
           },
+          "preset": {
+            "type": "string",
+            "enum": [
+              "balanced",
+              "custom"
+            ]
+          },
           "config_yaml": {
             "type": "string"
           },

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -5130,6 +5130,11 @@ components:
           type: string
         config:
           type: object
+        preset:
+          type: string
+          enum:
+            - balanced
+            - custom
         config_yaml:
           type: string
         ssl:
@@ -5451,6 +5456,11 @@ components:
           type: string
         config:
           type: object
+        preset:
+          type: string
+          enum:
+            - balanced
+            - custom
         config_yaml:
           type: string
         ssl:

x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml

@@ -22,6 +22,9 @@ properties:
     type: string
   config:
     type: object
+  preset:
+    type: string
+    enum: ['balanced', 'custom']
   config_yaml:
     type: string
   ssl:

x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml

@@ -22,6 +22,9 @@ properties:
     type: string
   config:
     type: object
+  preset:
+    type: string
+    enum: ['balanced', 'custom']
   config_yaml:
     type: string
   ssl:

x-pack/plugins/fleet/common/services/output_helpers.ts

@@ -5,12 +5,14 @@
  * 2.0.
  */
 
-import type { AgentPolicy } from '../types';
+import type { AgentPolicy, OutputType, ValueOf } from '../types';
 import {
   FLEET_APM_PACKAGE,
   FLEET_SERVER_PACKAGE,
   FLEET_SYNTHETICS_PACKAGE,
   outputType,
+  OUTPUT_TYPES_WITH_PRESET_SUPPORT,
+  RESERVED_CONFIG_YML_KEYS,
 } from '../constants';
 
 /**
@@ -33,3 +35,19 @@ export function getAllowedOutputTypeForPolicy(agentPolicy: AgentPolicy) {
 
   return Object.values(outputType);
 }
+
+export function outputYmlIncludesReservedPerformanceKey(configYml: string) {
+  return RESERVED_CONFIG_YML_KEYS.some((key) => configYml.includes(key));
+}
+
+export function getDefaultPresetForEsOutput(configYaml: string): 'balanced' | 'custom' {
+  if (outputYmlIncludesReservedPerformanceKey(configYaml)) {
+    return 'custom';
+  }
+
+  return 'balanced';
+}
+
+export function outputTypeSupportPresets(type: ValueOf<OutputType>) {
+  return OUTPUT_TYPES_WITH_PRESET_SUPPORT.includes(type);
+}

x-pack/plugins/fleet/common/types/models/output.ts

@@ -49,8 +49,11 @@ interface NewBaseOutput {
   shipper?: ShipperOutput | null;
   allow_edit?: string[];
   secrets?: {};
+  preset?: 'custom' | 'balanced';
 }
 
+export type OutputPreset = 'custom' | 'balanced' | 'throughput' | 'scale' | 'latency';
+
 export interface NewElasticsearchOutput extends NewBaseOutput {
   type: OutputType['Elasticsearch'];
 }

x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts

@@ -22,7 +22,9 @@ import {
   loadKafkaOutput,
   loadLogstashOutput,
   resetKafkaOutputForm,
+  selectESOutput,
   selectKafkaOutput,
+  selectRemoteESOutput,
   shouldDisplayError,
   validateOutputTypeChangeToKafka,
   validateSavedKafkaOutputForm,
@@ -37,6 +39,94 @@ describe('Outputs', () => {
     login();
   });
 
+  describe('Elasticsearch', () => {
+    describe('Preset input', () => {
+      it('is set to balanced by default', () => {
+        selectESOutput();
+
+        cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', 'balanced');
+      });
+
+      it('forces custom when reserved key is included in config YAML box', () => {
+        selectESOutput();
+
+        cy.getBySel('kibanaCodeEditor').click().focused().type('bulk_max_size: 1000');
+
+        cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT)
+          .should('have.value', 'custom')
+          .should('be.disabled');
+      });
+
+      it('allows balanced when reserved key is not included in config yaml box', () => {
+        selectESOutput();
+
+        cy.getBySel('kibanaCodeEditor').click().focused().type('some_random_key: foo');
+
+        cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT)
+          .should('have.value', 'balanced')
+          .should('be.enabled');
+      });
+
+      const cases = [
+        {
+          name: 'Preset: Balanced',
+          preset: 'balanced',
+        },
+        {
+          name: 'Preset: Custom',
+          preset: 'custom',
+        },
+        {
+          name: 'Preset: Throughput',
+          preset: 'throughput',
+        },
+        {
+          name: 'Preset: Scale',
+          preset: 'scale',
+        },
+        {
+          name: 'Preset: Latency',
+          preset: 'latency',
+        },
+      ];
+
+      for (const type of ['elasticsearch', 'remote_elasticsearch']) {
+        for (const testCase of cases) {
+          describe(`When type is ${type} and preset is ${testCase.name}`, () => {
+            it('successfully creates output', () => {
+              if (type === 'elasticsearch') {
+                selectESOutput();
+              } else if (type === 'remote_elasticsearch') {
+                selectRemoteESOutput();
+              }
+
+              cy.getBySel(SETTINGS_OUTPUTS.NAME_INPUT).type(testCase.name);
+              cy.get(`[placeholder="Specify host URL"]`).type(
+                `http://${testCase.preset}.elasticsearch.com:9200`
+              );
+              cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).select(testCase.preset);
+
+              cy.intercept('POST', '**/api/fleet/outputs').as('saveOutput');
+              cy.getBySel(SETTINGS_SAVE_BTN).click();
+
+              cy.wait('@saveOutput').then((interception) => {
+                const responseBody = interception.response?.body;
+                cy.visit(`/app/fleet/settings/outputs/${responseBody?.item?.id}`);
+              });
+
+              cy.getBySel(SETTINGS_OUTPUTS.NAME_INPUT).should('have.value', testCase.name);
+              cy.get(`[placeholder="Specify host URL"]`).should(
+                'have.value',
+                `http://${testCase.preset}.elasticsearch.com:9200`
+              );
+              cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', testCase.preset);
+            });
+          });
+        }
+      }
+    });
+  });
+
   describe('Kafka', () => {
     describe('Form validation', () => {
       it('renders all form fields', () => {
@@ -204,7 +294,7 @@ describe('Outputs', () => {
         cy.contains('Name is required');
         cy.contains('Host is required');
         cy.contains('Username is required');
-        // cy.contains('Password is required'); // TODO
+        cy.contains('Password is required');
         cy.contains('Default topic is required');
         cy.contains('Topic is required');
         cy.contains(
@@ -213,7 +303,7 @@ describe('Outputs', () => {
         cy.contains('Must be a key, value pair i.e. "http.response.code: 200"');
         shouldDisplayError(SETTINGS_OUTPUTS.NAME_INPUT);
         shouldDisplayError(SETTINGS_OUTPUTS_KAFKA.AUTHENTICATION_USERNAME_INPUT);
-        // shouldDisplayError(SETTINGS_OUTPUTS_KAFKA.AUTHENTICATION_PASSWORD_INPUT); // TODO
+        shouldDisplayError(SETTINGS_OUTPUTS_KAFKA.AUTHENTICATION_PASSWORD_INPUT);
         shouldDisplayError(SETTINGS_OUTPUTS_KAFKA.TOPICS_DEFAULT_TOPIC_INPUT);
         shouldDisplayError(SETTINGS_OUTPUTS_KAFKA.TOPICS_CONDITION_INPUT);
         shouldDisplayError(SETTINGS_OUTPUTS_KAFKA.TOPICS_TOPIC_INPUT);
@@ -271,7 +361,7 @@ describe('Outputs', () => {
       it('saves the output', () => {
         selectKafkaOutput();
 
-        fillInKafkaOutputForm(true);
+        fillInKafkaOutputForm();
 
         cy.intercept('POST', '**/api/fleet/outputs').as('saveOutput');
 

x-pack/plugins/fleet/cypress/screens/fleet.ts

@@ -124,6 +124,7 @@ export const SETTINGS_OUTPUTS = {
   ADD_HOST_ROW_BTN: 'fleetServerHosts.multiRowInput.addRowButton',
   WARNING_KAFKA_CALLOUT: 'settingsOutputsFlyout.kafkaOutputTypeCallout',
   WARNING_ELASTICSEARCH_CALLOUT: 'settingsOutputsFlyout.elasticsearchOutputTypeCallout',
+  PRESET_INPUT: 'settingsOutputsFlyout.presetInput',
 };
 
 export const getSpecificSelectorId = (selector: string, id: number) => {

x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts

@@ -15,6 +15,18 @@ import {
   SETTINGS_SAVE_BTN,
 } from './fleet';
 
+export const selectESOutput = () => {
+  visit('/app/fleet/settings');
+  cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click();
+  cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('elasticsearch');
+};
+
+export const selectRemoteESOutput = () => {
+  visit('/app/fleet/settings');
+  cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click();
+  cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('remote_elasticsearch');
+};
+
 export const selectKafkaOutput = () => {
   visit('/app/fleet/settings');
   cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click();