From 7692cfed954c1c04cf04eb6f5ba7218eb9b4e4dd Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Thu, 25 Jan 2024 15:09:27 +0100 Subject: [PATCH 01/10] add is_internal config option for outputs --- docs/settings/fleet-settings.asciidoc | 2 ++ x-pack/plugins/fleet/common/types/models/output.ts | 1 + .../fleet/public/applications/fleet/sections/settings/index.tsx | 2 +- x-pack/plugins/fleet/server/saved_objects/index.ts | 2 ++ x-pack/plugins/fleet/server/types/models/output.ts | 1 + x-pack/plugins/fleet/server/types/so_attributes.ts | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 110af39a5cc4c..f0ec598aa9eac 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -207,6 +207,8 @@ NOTE: The `xpack.fleet.outputs` settings are intended for advanced configuration If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent data unless there is another one configured specifically for the agent policy. `is_default_monitoring`::: If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent monitoring data unless there is another one configured specifically for the agent policy. + `is_internal`::: + If `true`, the output specified in `xpack.fleet.outputs` will be TBD `config`::: Extra config for that output. `proxy_id`::: diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 842ade5452963..0875eafd8fbdf 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -35,6 +35,7 @@ export type OutputPreset = 'custom' | 'balanced' | 'throughput' | 'scale' | 'lat interface NewBaseOutput { is_default: boolean; is_default_monitoring: boolean; + is_internal?: boolean; is_preconfigured?: boolean; name: string; type: ValueOf; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx index 420b1ef55ee11..04db65a559142 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx @@ -196,7 +196,7 @@ export const SettingsApp = withConfirmModalProvider(() => { !item.is_internal)} fleetServerHosts={fleetServerHosts.data.items} deleteOutput={deleteOutput} deleteFleetServerHost={deleteFleetServerHost} diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 82ffa8e762a5c..089a8d2222563 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -187,6 +187,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ config: { type: 'flattened' }, config_yaml: { type: 'text' }, is_preconfigured: { type: 'boolean', index: false }, + is_internal: { type: 'boolean', index: false }, ssl: { type: 'binary' }, proxy_id: { type: 'keyword' }, shipper: { @@ -699,6 +700,7 @@ export function registerEncryptedSavedObjects( 'ca_trusted_fingerprint', 'config', 'config_yaml', + 'is_internal', 'is_preconfigured', 'proxy_id', 'version', diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index f9e78b6db25a0..730ec512f5a0f 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -65,6 +65,7 @@ const BaseSchema = { name: schema.string(), is_default: schema.boolean({ defaultValue: false }), is_default_monitoring: schema.boolean({ defaultValue: false }), + is_internal: schema.maybe(schema.boolean()), ca_sha256: schema.maybe(schema.string()), ca_trusted_fingerprint: schema.maybe(schema.string()), config_yaml: schema.maybe(schema.string()), diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 1d6601e08e38f..09b49440fe3c3 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -138,6 +138,7 @@ interface OutputSoBaseAttributes { hosts?: string[]; ca_sha256?: string | null; ca_trusted_fingerprint?: string | null; + is_internal?: boolean; is_preconfigured?: boolean; config_yaml?: string | null; proxy_id?: string | null; From df81f0a290ea14b50bfc10b9d175db4502ab9f87 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Thu, 25 Jan 2024 17:49:29 +0100 Subject: [PATCH 02/10] add e2e test and disable internal outputs in the edit agent policy settings --- .../plugins/fleet/common/openapi/bundled.yaml | 14 +++++++++ .../output_create_request_elasticsearch.yaml | 2 ++ .../schemas/output_create_request_kafka.yaml | 2 ++ .../output_create_request_logstash.yaml | 2 ++ ...t_create_request_remote_elasticsearch.yaml | 2 ++ .../output_update_request_elasticsearch.yaml | 2 ++ .../schemas/output_update_request_kafka.yaml | 2 ++ .../output_update_request_logstash.yaml | 2 ++ .../cypress/e2e/fleet_settings_outputs.cy.ts | 30 +++++++++++++++++++ x-pack/plugins/fleet/cypress/screens/fleet.ts | 1 + .../agent_policy_advanced_fields/hooks.tsx | 7 +++-- .../components/outputs_table/index.tsx | 2 +- .../fleet/sections/settings/index.tsx | 7 +++-- 13 files changed, 69 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index a97c369fcb1bf..ab486bb2fb8c2 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -5121,6 +5121,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5189,6 +5191,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5337,6 +5341,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5407,6 +5413,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5450,6 +5458,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5520,6 +5530,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: @@ -5654,6 +5666,8 @@ components: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml index 0af1da40121d5..0cee8b3875f8d 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_elasticsearch.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml index 0a50abfb03d88..05617f26c949f 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_kafka.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml index a7142c967c4dc..b91a1473b8a6e 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_logstash.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml index 844b92df39b84..d05318820cda6 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml index 570d0da0138bf..b44cadd767dc4 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_elasticsearch.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml index 2ce5525a1a9f4..f7335071b87ba 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_kafka.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml index db6bdd12d8802..842d1c3c27867 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_update_request_logstash.yaml @@ -7,6 +7,8 @@ properties: type: boolean is_default_monitoring: type: boolean + is_internal: + type: boolean name: type: string type: diff --git a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts index 53e9461346106..6198611d6f304 100644 --- a/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts +++ b/x-pack/plugins/fleet/cypress/e2e/fleet_settings_outputs.cy.ts @@ -563,4 +563,34 @@ describe('Outputs', () => { }); }); }); + + describe('Outputs List', () => { + beforeEach(() => { + cy.intercept('/api/fleet/outputs', { + items: [ + { + id: 'fleet-default-output', + name: 'default', + type: 'elasticsearch', + is_default: true, + is_default_monitoring: true, + }, + { + id: 'internal-fleet-output', + name: 'internal output', + type: 'elasticsearch', + is_default: false, + is_default_monitoring: false, + is_internal: true, + }, + ], + }); + + cy.visit('/app/fleet/settings'); + }); + + it('should not display internal outputs', () => { + cy.getBySel(SETTINGS_OUTPUTS.TABLE).should('not.contain', 'internal output'); + }); + }); }); diff --git a/x-pack/plugins/fleet/cypress/screens/fleet.ts b/x-pack/plugins/fleet/cypress/screens/fleet.ts index 1d0acb2f34739..b48a720001c1b 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet.ts @@ -119,6 +119,7 @@ export const AGENT_BINARY_SOURCES_FLYOUT = { export const SETTINGS_OUTPUTS = { EDIT_BTN: 'editOutputBtn', ADD_BTN: 'addOutputBtn', + TABLE: 'settingsOutputsTable', NAME_INPUT: 'settingsOutputsFlyout.nameInput', TYPE_INPUT: 'settingsOutputsFlyout.typeInput', ADD_HOST_ROW_BTN: 'fleetServerHosts.multiRowInput.addRowButton', diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx index 15681ea1dfbb6..faa11129221f9 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx @@ -101,6 +101,7 @@ export function useOutputOptions(agentPolicy: Partial { const isOutputTypeUnsupported = !allowedOutputTypes.includes(item.type); + const isInternalOutput = item.is_internal; return { value: item.id, @@ -116,7 +117,7 @@ export function useOutputOptions(agentPolicy: Partial ) : undefined ), - disabled: !isPolicyPerOutputAllowed || isOutputTypeUnsupported, + disabled: !isPolicyPerOutputAllowed || isOutputTypeUnsupported || isInternalOutput, }; }), ]; @@ -133,10 +134,12 @@ export function useOutputOptions(agentPolicy: Partial { + const isInternalOutput = item.is_internal; + return { value: item.id, inputDisplay: item.name, - disabled: !isPolicyPerOutputAllowed, + disabled: !isPolicyPerOutputAllowed || isInternalOutput, }; }), ]; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx index dc27e83d881c1..84263d1730b04 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx @@ -164,5 +164,5 @@ export const OutputsTable: React.FunctionComponent = ({ ]; }, [deleteOutput, getHref]); - return ; + return ; }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx index 04db65a559142..889635a9df71c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx @@ -49,6 +49,7 @@ export const SettingsApp = withConfirmModalProvider(() => { const flyoutContext = useFlyoutContext(); const { outputs, fleetServerHosts, downloadSources, proxies } = useSettingsAppData(); + const outputItems = outputs.data?.items.filter((item) => !item.is_internal); const { deleteOutput } = useDeleteOutput(outputs.resendRequest); const { deleteDownloadSource } = useDeleteDownloadSource(downloadSources.resendRequest); @@ -78,7 +79,7 @@ export const SettingsApp = withConfirmModalProvider(() => { if ( (outputs.isLoading && outputs.isInitialRequest) || - !outputs.data?.items || + !outputItems || (fleetServerHosts.isLoading && fleetServerHosts.isInitialRequest) || !fleetServerHosts.data?.items || (downloadSources.isLoading && downloadSources.isInitialRequest) || @@ -148,7 +149,7 @@ export const SettingsApp = withConfirmModalProvider(() => { {(route: { match: { params: { outputId: string } } }) => { - const output = outputs.data?.items.find((o) => route.match.params.outputId === o.id); + const output = outputItems.find((o) => route.match.params.outputId === o.id); if (!output) { return ; } @@ -196,7 +197,7 @@ export const SettingsApp = withConfirmModalProvider(() => { !item.is_internal)} + outputs={outputItems} fleetServerHosts={fleetServerHosts.data.items} deleteOutput={deleteOutput} deleteFleetServerHost={deleteFleetServerHost} From aaedec85bdc985b7011816e64f2f41a3967c0028 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 26 Jan 2024 11:09:52 +0100 Subject: [PATCH 03/10] add is_internal flag to kbn-check-mappings-update-cli --- packages/kbn-check-mappings-update-cli/current_fields.json | 1 + packages/kbn-check-mappings-update-cli/current_mappings.json | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 3a51ffd206ab0..fff35c38cc103 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -524,6 +524,7 @@ "is_default", "is_default_monitoring", "is_preconfigured", + "is_internal", "key", "name", "output_id", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 1c4c15482934a..7a75cdcecc790 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1697,6 +1697,10 @@ "type": "boolean", "index": false }, + "is_internal": { + "type": "boolean", + "index": false + }, "ssl": { "type": "binary" }, From a3c170f806138b450e37db9909e1c814e89e0311 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:33:56 +0000 Subject: [PATCH 04/10] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- packages/kbn-check-mappings-update-cli/current_fields.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index fff35c38cc103..6a8b8c989ac70 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -523,8 +523,8 @@ "hosts", "is_default", "is_default_monitoring", - "is_preconfigured", "is_internal", + "is_preconfigured", "key", "name", "output_id", From 53cb801591f1242de417a28d55dc9e8ce3db777c Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 26 Jan 2024 13:56:58 +0100 Subject: [PATCH 05/10] fix so integration test --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 717c034a61b9d..f6a2919b163c9 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b", + "ingest-outputs": "39f71484e4fc30b5b9a261a9affb3044dcd6973d", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", From 8766ecf736168fb6c5631bbad1a82f3ba277a4b9 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 26 Jan 2024 14:11:20 +0100 Subject: [PATCH 06/10] fix isInternal checkin useOutputOptions hook --- .../components/agent_policy_advanced_fields/hooks.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx index faa11129221f9..5b36fd831bb65 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx @@ -101,7 +101,7 @@ export function useOutputOptions(agentPolicy: Partial { const isOutputTypeUnsupported = !allowedOutputTypes.includes(item.type); - const isInternalOutput = item.is_internal; + const isInternalOutput = !!item.is_internal; return { value: item.id, @@ -134,7 +134,7 @@ export function useOutputOptions(agentPolicy: Partial { - const isInternalOutput = item.is_internal; + const isInternalOutput = !!item.is_internal; return { value: item.id, From ea0c40d57d17cc3e8c066a794a18521bdeba79af Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 26 Jan 2024 14:22:11 +0100 Subject: [PATCH 07/10] add test for useOutputOptions to validate internal output handling --- .../hooks.test.tsx | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx index 8c2983ee9fd83..b9f653db37e1e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx @@ -120,6 +120,39 @@ const mockApiCallsWithRemoteESOutputs = (http: MockedFleetStartServices['http']) }); }; +const mockApiCallsWithInternalOutputs = (http: MockedFleetStartServices['http']) => { + http.get.mockImplementation(async (path) => { + if (typeof path !== 'string') { + throw new Error('Invalid request'); + } + if (path === '/api/fleet/outputs') { + return { + data: { + items: [ + { + id: 'default-output', + name: 'Default', + type: 'elasticsearch', + is_default: true, + is_default_monitoring: true, + }, + { + id: 'internal-output', + name: 'Internal', + type: 'elasticsearch', + is_default: false, + is_default_monitoring: false, + is_internal: true, + }, + ], + }, + }; + } + + return defaultHttpClientGetImplementation(path); + }); +}; + describe('useOutputOptions', () => { it('should generate enabled options if the licence is platinium', async () => { const testRenderer = createFleetTestRendererMock(); @@ -550,4 +583,56 @@ describe('useOutputOptions', () => { expect(result.current.monitoringOutputOptions.length).toEqual(2); expect(result.current.monitoringOutputOptions[1].value).toEqual('remote1'); }); + + it('should not enable internal outputs', async () => { + const testRenderer = createFleetTestRendererMock(); + mockedUseLicence.mockReturnValue({ + hasAtLeast: () => true, + } as unknown as LicenseService); + mockApiCallsWithInternalOutputs(testRenderer.startServices.http); + const { result, waitForNextUpdate } = testRenderer.renderHook(() => + useOutputOptions({} as AgentPolicy) + ); + expect(result.current.isLoading).toBeTruthy(); + + await waitForNextUpdate(); + expect(result.current.dataOutputOptions).toMatchInlineSnapshot(` + Array [ + Object { + "disabled": false, + "inputDisplay": "Default (currently Default)", + "value": "@@##DEFAULT_SELECT##@@", + }, + Object { + "disabled": false, + "inputDisplay": "Default", + "value": "default-output", + }, + Object { + "disabled": true, + "inputDisplay": "Internal", + "value": "internal-output", + }, + ] + `); + expect(result.current.monitoringOutputOptions).toMatchInlineSnapshot(` + Array [ + Object { + "disabled": undefined, + "inputDisplay": "Default (currently Default)", + "value": "@@##DEFAULT_SELECT##@@", + }, + Object { + "disabled": false, + "inputDisplay": "Default", + "value": "default-output", + }, + Object { + "disabled": true, + "inputDisplay": "Internal", + "value": "internal-output", + }, + ] + `); + }); }); From 9fb0ff5afe8452452f64aec7bb71346305e0d2a1 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 26 Jan 2024 14:27:26 +0100 Subject: [PATCH 08/10] add is_internal description for fleet settings docs --- docs/settings/fleet-settings.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index f0ec598aa9eac..02c5b1ed234c8 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -208,7 +208,7 @@ NOTE: The `xpack.fleet.outputs` settings are intended for advanced configuration `is_default_monitoring`::: If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent monitoring data unless there is another one configured specifically for the agent policy. `is_internal`::: - If `true`, the output specified in `xpack.fleet.outputs` will be TBD + If `true`, the output specified in `xpack.fleet.outputs` will be protected from user manipulation in the UI. `config`::: Extra config for that output. `proxy_id`::: From 806259bf060ebb6f8476f59e9c0e87c2f4a4e106 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Fri, 26 Jan 2024 17:15:03 +0100 Subject: [PATCH 09/10] Update docs wording for is_internal flag Co-authored-by: Kyle Pollich --- docs/settings/fleet-settings.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 02c5b1ed234c8..468cef7ad90f9 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -208,7 +208,7 @@ NOTE: The `xpack.fleet.outputs` settings are intended for advanced configuration `is_default_monitoring`::: If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent monitoring data unless there is another one configured specifically for the agent policy. `is_internal`::: - If `true`, the output specified in `xpack.fleet.outputs` will be protected from user manipulation in the UI. + If `true`, the output specified in `xpack.fleet.outputs` will not appear in the UI, and can only be managed via `kibana.yml` or the Fleet API. `config`::: Extra config for that output. `proxy_id`::: From bf9567c936c274e15fb5fd89a988cdb7f1214666 Mon Sep 17 00:00:00 2001 From: Maxim Kholod Date: Mon, 29 Jan 2024 18:00:55 +0100 Subject: [PATCH 10/10] add is_internal to fleet server host configuration --- docs/settings/fleet-settings.asciidoc | 2 ++ .../plugins/fleet/common/openapi/bundled.json | 30 +++++++++++++++++++ .../plugins/fleet/common/openapi/bundled.yaml | 6 ++++ .../components/schemas/fleet_server_host.yaml | 2 ++ .../openapi/paths/fleet_server_hosts.yaml | 2 ++ .../paths/fleet_server_hosts@{item_id}.yaml | 2 ++ .../models/fleet_server_policy_config.ts | 1 + .../types/rest_spec/fleet_server_hosts.ts | 2 ++ .../agent_policy_advanced_fields/hooks.tsx | 3 ++ .../fleet/sections/settings/index.tsx | 7 +++-- .../fleet/server/saved_objects/index.ts | 1 + .../preconfiguration/fleet_server_host.ts | 1 + .../server/types/models/preconfiguration.ts | 1 + .../rest_spec/fleet_server_policy_config.ts | 2 ++ .../fleet/server/types/so_attributes.ts | 1 + 15 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 468cef7ad90f9..6d3bb247e6ed4 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -269,6 +269,8 @@ List of {fleet-server} hosts that are configured when the {fleet} app starts. ===== `is_default`::: Whether or not this host should be the default to use for {fleet-server}. + `is_internal`::: + If `true` the host will not appear in the UI, and can only be managed via `kibana.yml` or the Fleet API. `proxy_id`::: Unique ID of the proxy to access the {fleet-server} host. ===== diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 0731047098a75..008e1d1f0ba37 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -5083,6 +5083,9 @@ "is_default": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "host_urls": { "type": "array", "items": { @@ -5195,6 +5198,9 @@ "is_default": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "host_urls": { "type": "array", "items": { @@ -7936,6 +7942,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8037,6 +8046,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8260,6 +8272,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8366,6 +8381,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8436,6 +8454,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8539,6 +8560,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8742,6 +8766,9 @@ "is_default_monitoring": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -8883,6 +8910,9 @@ "is_default": { "type": "boolean" }, + "is_internal": { + "type": "boolean" + }, "is_preconfigured": { "type": "boolean" }, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index ab486bb2fb8c2..849c22b47069a 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -3171,6 +3171,8 @@ paths: type: string is_default: type: boolean + is_internal: + type: boolean host_urls: type: array items: @@ -3241,6 +3243,8 @@ paths: type: string is_default: type: boolean + is_internal: + type: boolean host_urls: type: array items: @@ -5762,6 +5766,8 @@ components: type: string is_default: type: boolean + is_internal: + type: boolean is_preconfigured: type: boolean host_urls: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/fleet_server_host.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/fleet_server_host.yaml index 133bc7fcce13c..2bb08f3acc2a3 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/fleet_server_host.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/fleet_server_host.yaml @@ -7,6 +7,8 @@ properties: type: string is_default: type: boolean + is_internal: + type: boolean is_preconfigured: type: boolean host_urls: diff --git a/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts.yaml b/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts.yaml index d7668f3683b7b..987f61cd7619b 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts.yaml @@ -51,6 +51,8 @@ post: type: string is_default: type: boolean + is_internal: + type: boolean host_urls: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml b/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml index d46a8b86fb7f6..21d5342d18a5e 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml @@ -59,6 +59,8 @@ put: type: string is_default: type: boolean + is_internal: + type: boolean host_urls: type: array items: diff --git a/x-pack/plugins/fleet/common/types/models/fleet_server_policy_config.ts b/x-pack/plugins/fleet/common/types/models/fleet_server_policy_config.ts index 5609853a4ad2e..1e634df2b766d 100644 --- a/x-pack/plugins/fleet/common/types/models/fleet_server_policy_config.ts +++ b/x-pack/plugins/fleet/common/types/models/fleet_server_policy_config.ts @@ -11,6 +11,7 @@ export interface NewFleetServerHost { host_urls: string[]; is_default: boolean; is_preconfigured: boolean; + is_internal?: boolean; proxy_id?: string | null; } diff --git a/x-pack/plugins/fleet/common/types/rest_spec/fleet_server_hosts.ts b/x-pack/plugins/fleet/common/types/rest_spec/fleet_server_hosts.ts index 4a334c7e71b62..f07e96a0b927b 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/fleet_server_hosts.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/fleet_server_hosts.ts @@ -19,6 +19,7 @@ export interface PutFleetServerHostsRequest { name?: string; host_urls?: string[]; is_default?: boolean; + is_internal?: boolean; proxy_id?: string | null; }; } @@ -29,6 +30,7 @@ export interface PostFleetServerHostsRequest { name?: string; host_urls?: string[]; is_default?: boolean; + is_internal?: boolean; proxy_id?: string | null; }; } diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx index 5b36fd831bb65..7409f75a593dd 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx @@ -224,9 +224,12 @@ export function useFleetServerHostsOptions(agentPolicy: Partial !item.is_default) .map((item) => { + const isInternalFleetServerHost = !!item.is_internal; + return { value: item.id, inputDisplay: item.name, + disabled: isInternalFleetServerHost, }; }), ]; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx index 889635a9df71c..93f541d2ffd8b 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/index.tsx @@ -50,6 +50,7 @@ export const SettingsApp = withConfirmModalProvider(() => { const { outputs, fleetServerHosts, downloadSources, proxies } = useSettingsAppData(); const outputItems = outputs.data?.items.filter((item) => !item.is_internal); + const fleetServerHostsItems = fleetServerHosts.data?.items.filter((item) => !item.is_internal); const { deleteOutput } = useDeleteOutput(outputs.resendRequest); const { deleteDownloadSource } = useDeleteDownloadSource(downloadSources.resendRequest); @@ -81,7 +82,7 @@ export const SettingsApp = withConfirmModalProvider(() => { (outputs.isLoading && outputs.isInitialRequest) || !outputItems || (fleetServerHosts.isLoading && fleetServerHosts.isInitialRequest) || - !fleetServerHosts.data?.items || + !fleetServerHostsItems || (downloadSources.isLoading && downloadSources.isInitialRequest) || !downloadSources.data?.items || (proxies.isLoading && proxies.isInitialRequest) || @@ -99,7 +100,7 @@ export const SettingsApp = withConfirmModalProvider(() => { {(route: { match: { params: { itemId: string } } }) => { - const fleetServerHost = fleetServerHosts.data?.items.find( + const fleetServerHost = fleetServerHostsItems.find( (o) => route.match.params.itemId === o.id ); if (!fleetServerHost) { @@ -198,7 +199,7 @@ export const SettingsApp = withConfirmModalProvider(() => { deleteFleetProxy={deleteFleetProxy} proxies={proxies.data.items} outputs={outputItems} - fleetServerHosts={fleetServerHosts.data.items} + fleetServerHosts={fleetServerHostsItems} deleteOutput={deleteOutput} deleteFleetServerHost={deleteFleetServerHost} downloadSources={downloadSources.data.items} diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 089a8d2222563..8ed939c321324 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -616,6 +616,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ properties: { name: { type: 'keyword' }, is_default: { type: 'boolean' }, + is_internal: { type: 'boolean', index: false }, host_urls: { type: 'keyword', index: false }, is_preconfigured: { type: 'boolean' }, proxy_id: { type: 'keyword' }, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts b/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts index f622b1115e16e..5959cc25288ce 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts @@ -110,6 +110,7 @@ export async function createOrUpdatePreconfiguredFleetServerHosts( (!existingHost.is_preconfigured || existingHost.is_default !== preconfiguredFleetServerHost.is_default || existingHost.name !== preconfiguredFleetServerHost.name || + isDifferent(existingHost.is_internal, preconfiguredFleetServerHost.is_internal) || isDifferent( existingHost.host_urls.map(normalizeHostsForAgents), preconfiguredFleetServerHost.host_urls.map(normalizeHostsForAgents) diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 7572da134bd99..e814616268d3d 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -103,6 +103,7 @@ export const PreconfiguredFleetServerHostsSchema = schema.arrayOf( id: schema.string(), name: schema.string(), is_default: schema.boolean({ defaultValue: false }), + is_internal: schema.maybe(schema.boolean()), host_urls: schema.arrayOf(schema.string(), { minSize: 1 }), proxy_id: schema.nullable(schema.string()), }), diff --git a/x-pack/plugins/fleet/server/types/rest_spec/fleet_server_policy_config.ts b/x-pack/plugins/fleet/server/types/rest_spec/fleet_server_policy_config.ts index bf112a7d9abfe..d3f2f36a1624f 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/fleet_server_policy_config.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/fleet_server_policy_config.ts @@ -13,6 +13,7 @@ export const PostFleetServerHostRequestSchema = { name: schema.string(), host_urls: schema.arrayOf(schema.string(), { minSize: 1 }), is_default: schema.boolean({ defaultValue: false }), + is_internal: schema.maybe(schema.boolean()), proxy_id: schema.nullable(schema.string()), }), }; @@ -27,6 +28,7 @@ export const PutFleetServerHostRequestSchema = { name: schema.maybe(schema.string()), host_urls: schema.maybe(schema.arrayOf(schema.string(), { minSize: 1 })), is_default: schema.maybe(schema.boolean({ defaultValue: false })), + is_internal: schema.maybe(schema.boolean()), proxy_id: schema.nullable(schema.string()), }), }; diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 09b49440fe3c3..05b2cc1b05bae 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -104,6 +104,7 @@ export interface FleetServerHostSOAttributes { host_urls: string[]; is_default: boolean; is_preconfigured: boolean; + is_internal?: boolean; proxy_id?: string | null; }