[Fleet] Support Elasticsearch output performance presets (#172359)

## Summary

Closes #166870
Closes #172525

- Adds a new `preset` field to output saved objects
- Updates REST spec payloads to allow `preset` field in `POST/PUT`
requests to the `/api/fleet/outputs` endpoint
- Adds logic to set default `preset` to `balanced` or `custom` based on
whether a reserved key exists in `output.config_yaml`
- Adds UI to the output settings flyout for providing a preset
- Adds backfill logic to Fleet `setup` that updates all existing outputs
+ redeploys their associated policies to ensure the proper `preset` is
provided on all policies

## To do

- [x] Fix failing tests
- [x] Add a lot of tests + testing instructions
- [x] Allow preconfigured outputs to specify a preset
- [x] Update OpenAPI spec for outputs API
- [x] Disable `EuiSelect` when output is managed
- [x] Add in-product link to performance preset docs once they exist
(might have to be a follow-up? (Follow up:
#172523)
- [x] Parse YML box contents instead of using basic string lookup for
forcing `custom` preset (Follow up:
#172525)

## How to test

1. Create a new Elasticsearch output
2. Observe the `Performance preset` dropdown defaults to `balanced`
3. Add a performance setting to the custom YAML box e.g. `bulk_max_size:
1000`
4. Note the callout with the list of reserved keys
5. Note that the dropdown switches to `Custom` and is now disabled
6. Remove the offending key
7. Note the dropdown returns to its normal state
8. Save the output
9. Edit the output and observe the same behaviors

For the backfill
1. Create a local environment with multiple elasticsearch outputs on
`main`
2. Stop Kibana
3. Checkout this PR branch
4. Restart Kibana
5. Observe the ES outputs have been updated to include the appropriate
`preset` value

## Screenshots + Screen recordings


https://github.com/elastic/kibana/assets/6766512/0c25a15e-938d-4747-8846-d51a9ad01968

---------

Co-authored-by: kibanamachine <[email protected]>

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -528,6 +528,7 @@
     "output_id",
     "partition",
     "password",
+    "preset",
     "proxy_id",
     "random",
     "random.group_events",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1854,6 +1854,10 @@
             }
           }
         }
+      },
+      "preset": {
+        "type": "keyword",
+        "index": false
       }
     }
   },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
-        "ingest-outputs": "20bd44ce6016079c3b28f1b2bc241e7715be48f8",
+        "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b",
         "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43",
         "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",

x-pack/plugins/fleet/common/constants/output.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { NewOutput } from '../types';
+import type { NewOutput, OutputType, ValueOf } from '../types';
 
 export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs';
 
@@ -120,4 +120,19 @@ export const kafkaSupportedVersions = [
   '2.6.0',
 ];
 
+export const RESERVED_CONFIG_YML_KEYS = [
+  'bulk_max_size',
+  'workers',
+  'queue.mem.events',
+  'flush.min_events',
+  'flush.timeout',
+  'compression',
+  'idle_timeout',
+];
+
+export const OUTPUT_TYPES_WITH_PRESET_SUPPORT: Array<ValueOf<OutputType>> = [
+  outputType.Elasticsearch,
+  outputType.RemoteElasticsearch,
+];
+
 export const OUTPUT_HEALTH_DATA_STREAM = 'logs-fleet_server.output_health-default';

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -7952,6 +7952,16 @@
           "config": {
             "type": "object"
           },
+          "preset": {
+            "type": "string",
+            "enum": [
+              "balanced",
+              "custom",
+              "throughput",
+              "scale",
+              "latency"
+            ]
+          },
           "config_yaml": {
             "type": "string"
           },
@@ -8442,6 +8452,16 @@
           "config": {
             "type": "object"
           },
+          "preset": {
+            "type": "string",
+            "enum": [
+              "balanced",
+              "custom",
+              "throughput",
+              "scale",
+              "latency"
+            ]
+          },
           "config_yaml": {
             "type": "string"
           },

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -5130,6 +5130,14 @@ components:
           type: string
         config:
           type: object
+        preset:
+          type: string
+          enum:
+            - balanced
+            - custom
+            - throughput
+            - scale
+            - latency
         config_yaml:
           type: string
         ssl:
@@ -5451,6 +5459,14 @@ components:
           type: string
         config:
           type: object
+        preset:
+          type: string
+          enum:
+            - balanced
+            - custom
+            - throughput
+            - scale
+            - latency
         config_yaml:
           type: string
         ssl:

x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml

@@ -22,6 +22,9 @@ properties:
     type: string
   config:
     type: object
+  preset:
+    type: string
+    enum: ['balanced', 'custom', 'throughput', 'scale', 'latency']
   config_yaml:
     type: string
   ssl:

x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml

@@ -22,6 +22,9 @@ properties:
     type: string
   config:
     type: object
+  preset:
+    type: string
+    enum: ['balanced', 'custom', 'throughput', 'scale', 'latency']
   config_yaml:
     type: string
   ssl:

x-pack/plugins/fleet/common/services/output_helpers.test.ts

@@ -5,7 +5,12 @@
  * 2.0.
  */
 
-import { getAllowedOutputTypeForPolicy } from './output_helpers';
+import { safeLoad } from 'js-yaml';
+
+import {
+  getAllowedOutputTypeForPolicy,
+  outputYmlIncludesReservedPerformanceKey,
+} from './output_helpers';
 
 describe('getAllowedOutputTypeForPolicy', () => {
   it('should return all available output type for an agent policy without APM and Fleet Server', () => {
@@ -45,3 +50,55 @@ describe('getAllowedOutputTypeForPolicy', () => {
     expect(res).toEqual(['elasticsearch']);
   });
 });
+
+describe('outputYmlIncludesReservedPerformanceKey', () => {
+  describe('dot notation', () => {
+    it('returns true when reserved key is present', () => {
+      const configYml = `queue.mem.events: 1000`;
+
+      expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true);
+    });
+
+    it('returns false when no reserved key is present', () => {
+      const configYml = `some.random.key: 1000`;
+
+      expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false);
+    });
+  });
+
+  describe('object notation', () => {
+    it('returns true when reserved key is present', () => {
+      const configYml = `
+      queue:
+          mem:
+            events: 1000
+    `;
+
+      expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true);
+    });
+
+    it('returns false when no reserved key is present', () => {
+      const configYml = `
+        some:
+          random:
+            key: 1000
+      `;
+
+      expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false);
+    });
+  });
+
+  describe('plain string', () => {
+    it('returns true when reserved key is present', () => {
+      const configYml = `bulk_max_size`;
+
+      expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(true);
+    });
+
+    it('returns false when no reserved key is present', () => {
+      const configYml = `just a string`;
+
+      expect(outputYmlIncludesReservedPerformanceKey(configYml, safeLoad)).toBe(false);
+    });
+  });
+});

x-pack/plugins/fleet/common/services/output_helpers.ts

@@ -5,12 +5,18 @@
  * 2.0.
  */
 
-import type { AgentPolicy } from '../types';
+import { isObject } from 'lodash';
+
+import { getFlattenedObject } from '@kbn/std';
+
+import type { AgentPolicy, OutputType, ValueOf } from '../types';
 import {
   FLEET_APM_PACKAGE,
   FLEET_SERVER_PACKAGE,
   FLEET_SYNTHETICS_PACKAGE,
   outputType,
+  OUTPUT_TYPES_WITH_PRESET_SUPPORT,
+  RESERVED_CONFIG_YML_KEYS,
 } from '../constants';
 
 /**
@@ -33,3 +39,38 @@ export function getAllowedOutputTypeForPolicy(agentPolicy: AgentPolicy) {
 
   return Object.values(outputType);
 }
+
+export function outputYmlIncludesReservedPerformanceKey(
+  configYml: string,
+  // Dependency injection for `safeLoad` prevents bundle size issues 🤷‍♀️
+  safeLoad: (yml: string) => any
+) {
+  if (!configYml || configYml === '') {
+    return false;
+  }
+
+  const parsedYml = safeLoad(configYml);
+
+  if (!isObject(parsedYml)) {
+    return RESERVED_CONFIG_YML_KEYS.some((key) => parsedYml.includes(key));
+  }
+
+  const flattenedYml = isObject(parsedYml) ? getFlattenedObject(parsedYml) : {};
+
+  return RESERVED_CONFIG_YML_KEYS.some((key) => Object.keys(flattenedYml).includes(key));
+}
+
+export function getDefaultPresetForEsOutput(
+  configYaml: string,
+  safeLoad: (yml: string) => any
+): 'balanced' | 'custom' {
+  if (outputYmlIncludesReservedPerformanceKey(configYaml, safeLoad)) {
+    return 'custom';
+  }
+
+  return 'balanced';
+}
+
+export function outputTypeSupportPresets(type: ValueOf<OutputType>) {
+  return OUTPUT_TYPES_WITH_PRESET_SUPPORT.includes(type);
+}

x-pack/plugins/fleet/common/types/models/output.ts

@@ -29,6 +29,9 @@ export type OutputSecret =
       id: string;
       hash?: string;
     };
+
+export type OutputPreset = 'custom' | 'balanced' | 'throughput' | 'scale' | 'latency';
+
 interface NewBaseOutput {
   is_default: boolean;
   is_default_monitoring: boolean;
@@ -49,6 +52,7 @@ interface NewBaseOutput {
   shipper?: ShipperOutput | null;
   allow_edit?: string[];
   secrets?: {};
+  preset?: OutputPreset;
 }
 
 export interface NewElasticsearchOutput extends NewBaseOutput {