diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 110af39a5cc4c..468cef7ad90f9 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -207,6 +207,8 @@ NOTE: The `xpack.fleet.outputs` settings are intended for advanced configuration If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent data unless there is another one configured specifically for the agent policy. `is_default_monitoring`::: If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent monitoring data unless there is another one configured specifically for the agent policy. + `is_internal`::: + If `true`, the output specified in `xpack.fleet.outputs` will not appear in the UI, and can only be managed via `kibana.yml` or the Fleet API. `config`::: Extra config for that output. `proxy_id`::: diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 3a51ffd206ab0..6a8b8c989ac70 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -523,6 +523,7 @@ "hosts", "is_default", "is_default_monitoring", + "is_internal", "is_preconfigured", "key", "name", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 1c4c15482934a..7a75cdcecc790 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1697,6 +1697,10 @@ "type": "boolean", "index": false }, + "is_internal": { + "type": "boolean", + "index": false + }, "ssl": { "type": "binary" }, diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 717c034a61b9d..978cba8a06016 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b", + "ingest-outputs": "ba8ef97414bc983efdf1c4285afa622df8b4344a", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 0731047098a75..c04f0da6a3d85 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7936,6 +7936,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8037,6 +8040,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8260,6 +8266,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8366,6 +8375,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8436,6 +8448,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8539,6 +8554,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8742,6 +8760,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index a97c369fcb1bf..ab486bb2fb8c2 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -5121,6 +5121,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5189,6 +5191,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5337,6 +5341,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5407,6 +5413,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5450,6 +5458,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5520,6 +5530,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5654,6 +5666,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml index 0af1da40121d5..0cee8b3875f8d 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml index 0a50abfb03d88..05617f26c949f 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml index a7142c967c4dc..b91a1473b8a6e 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml index 844b92df39b84..d05318820cda6 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml index 570d0da0138bf..b44cadd767dc4 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml index 2ce5525a1a9f4..f7335071b87ba 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml index db6bdd12d8802..842d1c3c27867 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 842ade5452963..0875eafd8fbdf 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -35,6 +35,7 @@ export type OutputPreset = 'custom' | 'balanced' | 'throughput' | 'scale' | 'lat interface NewBaseOutput { is_default: boolean; is_default_monitoring: boolean; + is_internal?: boolean; is_preconfigured?: boolean; name: string; type: ValueOf; diff --git a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts index cd41b92cb6a8d..6aadca2da9d7e 100644 --- a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts @@ -594,4 +594,34 @@ queue: }); }); }); + + describe('Outputs List', () => { + beforeEach(() => { + cy.intercept('/api/fleet/outputs', { + items: [ + { + id: 'fleet-default-output', + name: 'default', + type: 'elasticsearch', + is_default: true, + is_default_monitoring: true, + }, + { + id: 'internal-fleet-output', + name: 'internal output', + type: 'elasticsearch', + is_default: false, + is_default_monitoring: false, + is_internal: true, + }, + ], + }); + + cy.visit('/app/fleet/settings'); + }); + + it('should not display internal outputs', () => { + cy.getBySel(SETTINGS_OUTPUTS.TABLE).should('not.contain', 'internal output'); + }); + }); }); diff --git a/x-pack/plugins/fleet/cypress/screens/fleet.ts b/x-pack/plugins/fleet/cypress/screens/fleet.ts index 1d0acb2f34739..b48a720001c1b 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet.ts @@ -119,6 +119,7 @@ export const AGENT_BINARY_SOURCES_FLYOUT = { export const SETTINGS_OUTPUTS = { EDIT_BTN: 'editOutputBtn', ADD_BTN: 'addOutputBtn', + TABLE: 'settingsOutputsTable', NAME_INPUT: 'settingsOutputsFlyout.nameInput', TYPE_INPUT: 'settingsOutputsFlyout.typeInput', ADD_HOST_ROW_BTN: 'fleetServerHosts.multiRowInput.addRowButton', diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx index 8c2983ee9fd83..b9f653db37e1e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx @@ -120,6 +120,39 @@ const mockApiCallsWithRemoteESOutputs = (http: MockedFleetStartServices['http']) }); }; +const mockApiCallsWithInternalOutputs = (http: MockedFleetStartServices['http']) => { + http.get.mockImplementation(async (path) => { + if (typeof path !== 'string') { + throw new Error('Invalid request'); + } + if (path === '/api/fleet/outputs') { + return { + data: { + items: [ + { + id: 'default-output', + name: 'Default', + type: 'elasticsearch', + is_default: true, + is_default_monitoring: true, + }, + { + id: 'internal-output', + name: 'Internal', + type: 'elasticsearch', + is_default: false, + is_default_monitoring: false, + is_internal: true, + }, + ], + }, + }; + } + + return defaultHttpClientGetImplementation(path); + }); +}; + describe('useOutputOptions', () => { it('should generate enabled options if the licence is platinium', async () => { const testRenderer = createFleetTestRendererMock(); @@ -550,4 +583,56 @@ describe('useOutputOptions', () => { expect(result.current.monitoringOutputOptions.length).toEqual(2); expect(result.current.monitoringOutputOptions[1].value).toEqual('remote1'); }); + + it('should not enable internal outputs', async () => { + const testRenderer = createFleetTestRendererMock(); + mockedUseLicence.mockReturnValue({ + hasAtLeast: () => true, + } as unknown as LicenseService); + mockApiCallsWithInternalOutputs(testRenderer.startServices.http); + const { result, waitForNextUpdate } = testRenderer.renderHook(() => + useOutputOptions({} as AgentPolicy) + ); + expect(result.current.isLoading).toBeTruthy(); + + await waitForNextUpdate(); + expect(result.current.dataOutputOptions).toMatchInlineSnapshot(` + Array [ + Object { + "disabled": false, + "inputDisplay": "Default (currently Default)", + "value": "@@##DEFAULT_SELECT##@@", + }, + Object { + "disabled": false, + "inputDisplay": "Default", + "value": "default-output", + }, + Object { + "disabled": true, + "inputDisplay": "Internal", + "value": "internal-output", + }, + ] + `); + expect(result.current.monitoringOutputOptions).toMatchInlineSnapshot(` + Array [ + Object { + "disabled": undefined, + "inputDisplay": "Default (currently Default)", + "value": "@@##DEFAULT_SELECT##@@", + }, + Object { + "disabled": false, + "inputDisplay": "Default", + "value": "default-output", + }, + Object { + "disabled": true, + "inputDisplay": "Internal", + "value": "internal-output", + }, + ] + `); + }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx index 15681ea1dfbb6..5b36fd831bb65 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx @@ -101,6 +101,7 @@ export function useOutputOptions(agentPolicy: Partial { const isOutputTypeUnsupported = !allowedOutputTypes.includes(item.type); + const isInternalOutput = !!item.is_internal; return { value: item.id, @@ -116,7 +117,7 @@ export function useOutputOptions(agentPolicy: Partial ) : undefined ), - disabled: !isPolicyPerOutputAllowed || isOutputTypeUnsupported, + disabled: !isPolicyPerOutputAllowed || isOutputTypeUnsupported || isInternalOutput, }; }), ]; @@ -133,10 +134,12 @@ export function useOutputOptions(agentPolicy: Partial { + const isInternalOutput = !!item.is_internal; + return { value: item.id, inputDisplay: item.name, - disabled: !isPolicyPerOutputAllowed, + disabled: !isPolicyPerOutputAllowed || isInternalOutput, }; }), ]; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx index dc27e83d881c1..84263d1730b04 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx @@ -164,5 +164,5 @@ export const OutputsTable: React.FunctionComponent = ({ ]; }, [deleteOutput, getHref]); - return ; + return ; }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx index 420b1ef55ee11..889635a9df71c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx @@ -49,6 +49,7 @@ export const SettingsApp = withConfirmModalProvider(() => { const flyoutContext = useFlyoutContext(); const { outputs, fleetServerHosts, downloadSources, proxies } = useSettingsAppData(); + const outputItems = outputs.data?.items.filter((item) => !item.is_internal); const { deleteOutput } = useDeleteOutput(outputs.resendRequest); const { deleteDownloadSource } = useDeleteDownloadSource(downloadSources.resendRequest); @@ -78,7 +79,7 @@ export const SettingsApp = withConfirmModalProvider(() => { if ( (outputs.isLoading && outputs.isInitialRequest) || - !outputs.data?.items || + !outputItems || (fleetServerHosts.isLoading && fleetServerHosts.isInitialRequest) || !fleetServerHosts.data?.items || (downloadSources.isLoading && downloadSources.isInitialRequest) || @@ -148,7 +149,7 @@ export const SettingsApp = withConfirmModalProvider(() => { {(route: { match: { params: { outputId: string } } }) => { - const output = outputs.data?.items.find((o) => route.match.params.outputId === o.id); + const output = outputItems.find((o) => route.match.params.outputId === o.id); if (!output) { return ; } @@ -196,7 +197,7 @@ export const SettingsApp = withConfirmModalProvider(() => { ({ config: { type: 'flattened' }, config_yaml: { type: 'text' }, is_preconfigured: { type: 'boolean', index: false }, + is_internal: { type: 'boolean', index: false }, ssl: { type: 'binary' }, proxy_id: { type: 'keyword' }, shipper: { @@ -351,6 +352,16 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, ], }, + '5': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + is_internal: { type: 'boolean', index: false }, + }, + }, + ], + }, }, migrations: { '7.13.0': migrateOutputToV7130, @@ -699,6 +710,7 @@ export function registerEncryptedSavedObjects( 'ca_trusted_fingerprint', 'config', 'config_yaml', + 'is_internal', 'is_preconfigured', 'proxy_id', 'version', diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index 174b8051a9914..d640f74f4b03b 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -497,6 +497,27 @@ describe('output preconfiguration', () => { expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); }); + it('should update output if preconfigured output exists and changed to is_internal: true', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 }); + await createOrUpdatePreconfiguredOutputs(soClient, esClient, [ + { + id: 'existing-es-output-1', + is_default: false, + is_default_monitoring: false, + name: 'ES Output 1', + type: 'elasticsearch', + hosts: ['http://es.co:80'], + is_internal: true, + }, + ]); + + expect(mockedOutputService.create).not.toBeCalled(); + expect(mockedOutputService.update).toBeCalled(); + expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled(); + }); + it('should update output if a preconfigured logstash ouput exists and has changed', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index f73481ff402e3..4ee724f1a80b2 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -387,6 +387,7 @@ async function isPreconfiguredOutputDifferentFromCurrent( isDifferent(existingOutput.allow_edit ?? [], preconfiguredOutput.allow_edit ?? []) || (preconfiguredOutput.preset && isDifferent(existingOutput.preset, preconfiguredOutput.preset)) || + isDifferent(existingOutput.is_internal, preconfiguredOutput.is_internal) || (await kafkaFieldsAreDifferent()) || (await logstashFieldsAreDifferent()) || (await remoteESFieldsAreDifferent()) diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index f9e78b6db25a0..730ec512f5a0f 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -65,6 +65,7 @@ const BaseSchema = { name: schema.string(), is_default: schema.boolean({ defaultValue: false }), is_default_monitoring: schema.boolean({ defaultValue: false }), + is_internal: schema.maybe(schema.boolean()), ca_sha256: schema.maybe(schema.string()), ca_trusted_fingerprint: schema.maybe(schema.string()), config_yaml: schema.maybe(schema.string()), diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 1d6601e08e38f..09b49440fe3c3 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -138,6 +138,7 @@ interface OutputSoBaseAttributes { hosts?: string[]; ca_sha256?: string | null; ca_trusted_fingerprint?: string | null; + is_internal?: boolean; is_preconfigured?: boolean; config_yaml?: string | null; proxy_id?: string | null;