From 6a0c90025c16e68fb1d21bc5e44e051b840a3ea4 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 6 Nov 2024 15:30:03 -0500 Subject: [PATCH 1/7] [Fleet] Remove deprecated topics property for kafka output in favor of topic --- .../fleet/common/types/models/output.ts | 7 -- .../edit_output_flyout/index.test.tsx | 4 +- .../use_output_form.test.tsx | 64 ++++--------------- .../edit_output_flyout/use_output_form.tsx | 33 +++------- .../fleet/server/saved_objects/index.ts | 13 ++++ .../saved_objects/model_versions/outputs.ts | 32 ++++++++++ .../agent_policies/full_agent_policy.test.ts | 11 +--- .../agent_policies/full_agent_policy.ts | 5 +- .../fleet/server/services/output.test.ts | 4 +- .../plugins/fleet/server/services/output.ts | 1 - .../services/preconfiguration/outputs.ts | 1 - .../fleet/server/types/models/output.ts | 26 -------- .../types/models/preconfiguration.test.ts | 2 +- 13 files changed, 75 insertions(+), 128 deletions(-) create mode 100644 x-pack/plugins/fleet/server/saved_objects/model_versions/outputs.ts diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 36471c5e95bfd..b47a393013eda 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -127,13 +127,6 @@ export interface KafkaOutput extends NewBaseOutput { random?: boolean; }; topic?: string; - topics?: Array<{ - topic: string; - when?: { - type?: ValueOf; - condition?: string; - }; - }>; headers?: Array<{ key: string; value: string; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx index e742005a37913..2b343fcdcb574 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx @@ -207,7 +207,7 @@ describe('EditOutputFlyout', () => { is_default: false, is_default_monitoring: false, hosts: ['kafka:443'], - topics: [{ topic: 'topic' }], + topic: 'topic', auth_type: 'ssl', version: '1.0.0', ssl: { certificate: 'cert', key: 'key', verification_mode: 'full' }, @@ -247,7 +247,7 @@ describe('EditOutputFlyout', () => { is_default: false, is_default_monitoring: false, hosts: ['kafka:443'], - topics: [{ topic: 'topic' }], + topic: 'topic', auth_type: 'user_pass', version: '1.0.0', username: 'user', diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.test.tsx index cd0b04c33eaf2..e088b21a04c0a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.test.tsx @@ -12,7 +12,7 @@ import { describe('use_output_form', () => { describe('extractDefaultDynamicKafkaTopics', () => { - it('should return empty array if not topics are passed', () => { + it('should return empty array if not topic are passed', () => { const res = extractDefaultDynamicKafkaTopics({ type: 'kafka', name: 'new', @@ -23,37 +23,25 @@ describe('use_output_form', () => { expect(res).toEqual([]); }); - it('should return empty array if topics have length == 0', () => { + it('should return empty array if topic do not include %{[', () => { const res = extractDefaultDynamicKafkaTopics({ type: 'kafka', name: 'new', is_default: false, is_default_monitoring: false, - topics: [], + topic: 'something', }); expect(res).toEqual([]); }); - it('should return empty array if topics do not include %{[', () => { + it('should return options for combobox if topic include %{[', () => { const res = extractDefaultDynamicKafkaTopics({ type: 'kafka', name: 'new', is_default: false, is_default_monitoring: false, - topics: [{ topic: 'something' }], - }); - - expect(res).toEqual([]); - }); - - it('should return options for combobox if topics include %{[', () => { - const res = extractDefaultDynamicKafkaTopics({ - type: 'kafka', - name: 'new', - is_default: false, - is_default_monitoring: false, - topics: [{ topic: '%{[default.dataset]}' }], + topic: '%{[default.dataset]}', }); expect(res).toEqual([ @@ -64,13 +52,13 @@ describe('use_output_form', () => { ]); }); - it('should return options for combobox if topics include %{[ and some special characters', () => { + it('should return options for combobox if topic include %{[ and some special characters', () => { const res = extractDefaultDynamicKafkaTopics({ type: 'kafka', name: 'new', is_default: false, is_default_monitoring: false, - topics: [{ topic: '%{[@timestamp]}' }], + topic: '%{[@timestamp]}', }); expect(res).toEqual([ @@ -81,13 +69,13 @@ describe('use_output_form', () => { ]); }); - it('should return options for combobox if topics include %{[ and a custom name', () => { + it('should return options for combobox if topic include %{[ and a custom name', () => { const res = extractDefaultDynamicKafkaTopics({ type: 'kafka', name: 'new', is_default: false, is_default_monitoring: false, - topics: [{ topic: '%{[something]}' }], + topic: '%{[something]}', }); expect(res).toEqual([ @@ -100,7 +88,7 @@ describe('use_output_form', () => { }); describe('extractDefaultStaticKafkaTopic', () => { - it('should return empty array if not topics are passed', () => { + it('should return empty array if not topic are passed', () => { const res = extractDefaultStaticKafkaTopic({ type: 'kafka', name: 'new', @@ -111,52 +99,28 @@ describe('use_output_form', () => { expect(res).toEqual(''); }); - it('should return empty array if topics have length == 0', () => { + it('should return empty string if topic include %{[', () => { const res = extractDefaultStaticKafkaTopic({ type: 'kafka', name: 'new', is_default: false, is_default_monitoring: false, - topics: [], + topic: '%{[something]}', }); expect(res).toEqual(''); }); - it('should return empty string if topics include %{[', () => { + it('should return the topic if topic field is defined', () => { const res = extractDefaultStaticKafkaTopic({ type: 'kafka', name: 'new', is_default: false, is_default_monitoring: false, - topics: [{ topic: '%{[something]}' }], - }); - - expect(res).toEqual(''); - }); - - it('should return the topic if topics field is defined', () => { - const res = extractDefaultStaticKafkaTopic({ - type: 'kafka', - name: 'new', - is_default: false, - is_default_monitoring: false, - topics: [{ topic: 'something' }], + topic: 'something', }); expect(res).toEqual('something'); }); - - it('should return the last topic if topics field is defined and has multiple', () => { - const res = extractDefaultStaticKafkaTopic({ - type: 'kafka', - name: 'new', - is_default: false, - is_default_monitoring: false, - topics: [{ topic: 'something_1' }, { topic: 'something_2' }, { topic: 'something_3' }], - }); - - expect(res).toEqual('something_3'); - }); }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx index 8526fcdae001a..b75d41dd5744a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx @@ -162,25 +162,20 @@ function extractKafkaOutputSecrets( } export function extractDefaultStaticKafkaTopic(o: KafkaOutput): string { - if ( - !o?.topics || - o.topics?.length === 0 || - (o.topics && o?.topics.length > 0 && o.topics[0].topic?.includes('%{[')) - ) { + if (o.topic?.includes('%{[')) { return ''; } - const lastTopic = o.topics[o.topics.length - 1].topic; - return lastTopic || ''; + return o.topic || ''; } export function extractDefaultDynamicKafkaTopics( o: KafkaOutput ): Array> { - if (!o?.topics || o.topics?.length === 0 || (o.topics && !o.topics[0]?.topic?.includes('%{['))) { + if (!o?.topic || (o.topic && !o.topic?.includes('%{['))) { return []; } - const matched = o.topics[0].topic.match(/(%\{\[)(\S*)(\]\})/); + const matched = o.topic.match(/(%\{\[)(\S*)(\]\})/); const parsed = matched?.length ? matched[2] : ''; return [ @@ -470,23 +465,23 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu ); const kafkaTopicsInput = useRadioInput( - kafkaOutput?.topics && kafkaOutput?.topics[0].topic?.includes('%{[') + kafkaOutput?.topic && kafkaOutput?.topic?.includes('%{[') ? kafkaTopicsType.Dynamic : kafkaTopicsType.Static, - isDisabled('topics') + isDisabled('topic') ); const kafkaStaticTopicInput = useInput( extractDefaultStaticKafkaTopic(kafkaOutput), kafkaTopicsInput.value === kafkaTopicsType.Static ? validateKafkaStaticTopic : undefined, - isDisabled('topics') + isDisabled('topic') ); const kafkaDynamicTopicInput = useComboBoxWithCustomInput( 'kafkaDynamicTopicComboBox', extractDefaultDynamicKafkaTopics(kafkaOutput), kafkaTopicsInput.value === kafkaTopicsType.Dynamic ? validateDynamicKafkaTopics : undefined, - isDisabled('topics') + isDisabled('topic') ); const kafkaHeadersInput = useKeyValueInput( @@ -874,19 +869,11 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu : {}), ...(kafkaTopicsInput.value === kafkaTopicsType.Static && kafkaStaticTopicInput.value ? { - topics: [ - { - topic: kafkaStaticTopicInput.value, - }, - ], + topic: kafkaStaticTopicInput.value, } : kafkaTopicsInput.value === kafkaTopicsType.Dynamic && kafkaDynamicTopicInput.value ? { - topics: [ - { - topic: `%{[${kafkaDynamicTopicInput.value}]}`, - }, - ], + topic: `%{[${kafkaDynamicTopicInput.value}]}`, } : {}), headers: kafkaHeadersInput.value, diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index ffb9381f8b30c..706d0686b9845 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -101,6 +101,7 @@ import { migratePackagePolicySetRequiresRootToV8150, } from './migrations/to_v8_15_0'; import { backfillAgentPolicyToV4 } from './model_versions/agent_policy_v4'; +import { backfillOutputPolicyToV7 } from './model_versions/outputs'; /* * Saved object types and mappings @@ -558,6 +559,18 @@ export const getSavedObjectTypes = ( }, ], }, + '7': { + changes: [ + { + type: 'mappings_deprecation', + deprecatedMappings: ['topics'], + }, + { + type: 'data_backfill', + backfillFn: backfillOutputPolicyToV7, + }, + ], + }, }, migrations: { '7.13.0': migrateOutputToV7130, diff --git a/x-pack/plugins/fleet/server/saved_objects/model_versions/outputs.ts b/x-pack/plugins/fleet/server/saved_objects/model_versions/outputs.ts new file mode 100644 index 0000000000000..ed4e82d1c1ec1 --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/model_versions/outputs.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server'; + +import type { Output } from '../../../common'; + +export const backfillOutputPolicyToV7: SavedObjectModelDataBackfillFn< + Output & { + topics?: Array<{ + topic: string; + when?: { + type?: string; + condition?: string; + }; + }>; + }, + Output +> = (outputDoc) => { + if (outputDoc.attributes.type === 'kafka' && outputDoc.attributes.topics) { + if (!outputDoc.attributes.topic) { + outputDoc.attributes.topic = outputDoc.attributes.topics?.filter((t) => !t.when)?.[0]?.topic; + } + + delete outputDoc.attributes.topics; + } + return outputDoc; +}; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 609c560906de2..688320825326a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -1198,16 +1198,7 @@ ssl.test: 123 const policyOutput = transformOutputToFullPolicyOutput({ id: 'id123', hosts: ['test:9999'], - topics: [ - { - topic: 'test', - }, - // Deprecated conditionnal topic - { - topic: 'deprecated', - when: { condition: 'test:100', type: 'equals' }, - }, - ], + topic: 'test', is_default: false, is_default_monitoring: false, name: 'test output', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 6a586364f31d5..8116856265157 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -386,15 +386,12 @@ export function transformOutputToFullPolicyOutput( round_robin, hash, topic, - topics, headers, timeout, broker_timeout, required_acks, } = output; - const kafkaTopic = topic ? topic : topics?.filter((t) => !t.when)?.[0]?.topic; - const transformPartition = () => { if (!partition) return {}; switch (partition) { @@ -431,7 +428,7 @@ export function transformOutputToFullPolicyOutput( ...(password ? { password } : {}), ...(sasl ? { sasl } : {}), partition: transformPartition(), - topic: kafkaTopic, + topic, headers: (headers ?? []).filter((item) => item.key !== '' || item.value !== ''), timeout, broker_timeout, diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index d35701e77f882..f2dab8401e641 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -824,7 +824,7 @@ describe('Output Service', () => { is_default_monitoring: false, name: 'Test', type: 'kafka', - topics: [{ topic: 'test' }], + topic: 'test', }, { id: 'output-test' } ) @@ -1206,7 +1206,6 @@ describe('Output Service', () => { ssl: null, timeout: null, topic: null, - topics: null, headers: null, username: null, version: null, @@ -1330,7 +1329,6 @@ describe('Output Service', () => { sasl: null, timeout: null, topic: null, - topics: null, headers: null, username: null, version: null, diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index b5041d9f1df37..ceecd29562fc9 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -919,7 +919,6 @@ class OutputService { target.random = null; target.round_robin = null; target.hash = null; - target.topics = null; target.topic = null; target.headers = null; target.timeout = null; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 68b4cf2457e26..9d71e15958480 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -318,7 +318,6 @@ async function isPreconfiguredOutputDifferentFromCurrent( isDifferent(existingOutput.round_robin, preconfiguredOutput.round_robin) || isDifferent(existingOutput.hash, preconfiguredOutput.hash) || isDifferent(existingOutput.topic, preconfiguredOutput.topic) || - isDifferent(existingOutput.topics, preconfiguredOutput.topics) || isDifferent(existingOutput.headers, preconfiguredOutput.headers) || isDifferent(existingOutput.timeout, preconfiguredOutput.timeout) || isDifferent(existingOutput.broker_timeout, preconfiguredOutput.broker_timeout) || diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index 4385f9911e3db..2f02819554d53 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -13,7 +13,6 @@ import { kafkaConnectionType, kafkaPartitionType, kafkaSaslMechanism, - kafkaTopicWhenType, kafkaVerificationModes, outputType, } from '../../../common/constants'; @@ -196,29 +195,6 @@ const LogstashUpdateSchema = { ), }; -/** - * Kafka schemas - */ - -const KafkaTopicsSchema = schema.arrayOf( - schema.object({ - topic: schema.string(), - when: schema.maybe( - schema.object({ - type: schema.maybe( - schema.oneOf([ - schema.literal(kafkaTopicWhenType.Equals), - schema.literal(kafkaTopicWhenType.Contains), - schema.literal(kafkaTopicWhenType.Regexp), - ]) - ), - condition: schema.maybe(schema.string()), - }) - ), - }), - { minSize: 1 } -); - export const KafkaSchema = { ...BaseSchema, type: schema.literal(outputType.Kafka), @@ -303,7 +279,6 @@ export const KafkaSchema = { schema.object({ hash: schema.maybe(schema.string()), random: schema.maybe(schema.boolean()) }) ), topic: schema.maybe(schema.string()), - topics: schema.maybe(KafkaTopicsSchema), headers: schema.maybe( schema.arrayOf(schema.object({ key: schema.string(), value: schema.string() })) ), @@ -335,7 +310,6 @@ const KafkaUpdateSchema = { schema.literal(kafkaAuthType.Kerberos), ]) ), - topics: schema.maybe(KafkaTopicsSchema), }; export const OutputSchema = schema.oneOf([ diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts index a436f799fc171..a924ba7c8c7df 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts @@ -119,7 +119,7 @@ describe('Test preconfiguration schema', () => { type: 'kafka', hosts: ['localhost:9200'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', secrets: { password: 'mypassword', ssl: { From 6b87ca486f321e3c3bc31216700bf7999d296778 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 6 Nov 2024 20:45:40 +0000 Subject: [PATCH 2/7] [CI] Auto-commit changed files from 'node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules --include-path /api/actions --include-path /api/security/role --include-path /api/spaces --include-path /api/fleet --update' --- oas_docs/bundle.json | 198 -------------------------------- oas_docs/bundle.serverless.json | 198 -------------------------------- 2 files changed, 396 deletions(-) diff --git a/oas_docs/bundle.json b/oas_docs/bundle.json index 19094b82be094..cb748564415c2 100644 --- a/oas_docs/bundle.json +++ b/oas_docs/bundle.json @@ -23782,39 +23782,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -24917,39 +24884,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": false, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": false, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -25978,39 +25912,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -27213,39 +27114,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -28321,39 +28189,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": false, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": false, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -29379,39 +29214,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" diff --git a/oas_docs/bundle.serverless.json b/oas_docs/bundle.serverless.json index bc3d45fe67960..db6678511d529 100644 --- a/oas_docs/bundle.serverless.json +++ b/oas_docs/bundle.serverless.json @@ -23782,39 +23782,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -24917,39 +24884,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": false, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": false, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -25978,39 +25912,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -27213,39 +27114,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -28321,39 +28189,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": false, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": false, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" @@ -29379,39 +29214,6 @@ "topic": { "type": "string" }, - "topics": { - "items": { - "additionalProperties": true, - "properties": { - "topic": { - "type": "string" - }, - "when": { - "additionalProperties": true, - "properties": { - "condition": { - "type": "string" - }, - "type": { - "enum": [ - "equals", - "contains", - "regexp" - ], - "type": "string" - } - }, - "type": "object" - } - }, - "required": [ - "topic" - ], - "type": "object" - }, - "minItems": 1, - "type": "array" - }, "type": { "enum": [ "kafka" From 0de5ac739df64f8abee8e7be2a236fb0cd9dd9b6 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 6 Nov 2024 20:49:39 +0000 Subject: [PATCH 3/7] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index f31ec223e3d9b..cf0b0df1a8a8a 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -122,7 +122,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "5e95e539826a40ad08fd0c1d161da0a4d86ffc6d", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91", + "ingest-outputs": "55988d5f778bbe0e76caa7e6468707a0a056bdd8", "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", From 8b0747beff587595359575152dc5b098a177ee92 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 6 Nov 2024 23:26:15 +0000 Subject: [PATCH 4/7] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.serverless.yaml | 138 ------------------------- oas_docs/output/kibana.yaml | 138 ------------------------- 2 files changed, 276 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 78c8541059d26..92a7d24182948 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -22419,29 +22419,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -23176,29 +23153,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: false - type: object - properties: - topic: - type: string - when: - additionalProperties: false - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -23888,29 +23842,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -24710,29 +24641,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -25444,29 +25352,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: false - type: object - properties: - topic: - type: string - when: - additionalProperties: false - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -26153,29 +26038,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 64d227b91979d..741849de09568 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -25852,29 +25852,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -26609,29 +26586,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: false - type: object - properties: - topic: - type: string - when: - additionalProperties: false - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -27321,29 +27275,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -28143,29 +28074,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -28877,29 +28785,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: false - type: object - properties: - topic: - type: string - when: - additionalProperties: false - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka @@ -29586,29 +29471,6 @@ paths: type: number topic: type: string - topics: - items: - additionalProperties: true - type: object - properties: - topic: - type: string - when: - additionalProperties: true - type: object - properties: - condition: - type: string - type: - enum: - - equals - - contains - - regexp - type: string - required: - - topic - minItems: 1 - type: array type: enum: - kafka From 25d93be538b1ff23293597316e22a2ba04c72414 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 7 Nov 2024 07:51:07 -0500 Subject: [PATCH 5/7] fix typo --- .../components/edit_output_flyout/use_output_form.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx index b75d41dd5744a..e47e271872309 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx @@ -162,17 +162,17 @@ function extractKafkaOutputSecrets( } export function extractDefaultStaticKafkaTopic(o: KafkaOutput): string { - if (o.topic?.includes('%{[')) { + if (o?.topic?.includes('%{[')) { return ''; } - return o.topic || ''; + return o?.topic || ''; } export function extractDefaultDynamicKafkaTopics( o: KafkaOutput ): Array> { - if (!o?.topic || (o.topic && !o.topic?.includes('%{['))) { + if (!o?.topic || (o?.topic && !o.topic?.includes('%{['))) { return []; } const matched = o.topic.match(/(%\{\[)(\S*)(\]\})/); From 2e69756f69603127b60e0a4ddb000403f49a608e Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 7 Nov 2024 07:58:23 -0500 Subject: [PATCH 6/7] fix tests --- .../apis/outputs/crud.ts | 32 +++++++++---------- .../apis/policy_secrets.ts | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 564259cb49a23..2ba46620ef0d3 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -445,7 +445,7 @@ export default function (providerContext: FtrProviderContext) { password: 'pass', is_default: true, is_default_monitoring: true, - topics: [{ topic: 'topic1' }], + topic: 'topic1', }) .expect(400); expect(body.message).to.eql( @@ -727,7 +727,7 @@ export default function (providerContext: FtrProviderContext) { type: 'kafka', hosts: ['test.fr:2000'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -760,7 +760,7 @@ export default function (providerContext: FtrProviderContext) { type: 'kafka', hosts: ['test.fr:2000'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1107,7 +1107,7 @@ export default function (providerContext: FtrProviderContext) { auth_type: 'user_pass', username: 'user', password: 'pass', - topics: [{ topic: 'topic1' }], + topic: 'topic1', }) .expect(200); @@ -1121,7 +1121,7 @@ export default function (providerContext: FtrProviderContext) { auth_type: 'user_pass', username: 'user', password: 'pass', - topics: [{ topic: 'topic1' }], + topic: 'topic1', broker_timeout: 10, required_acks: 1, client_id: 'Elastic', @@ -1147,7 +1147,7 @@ export default function (providerContext: FtrProviderContext) { auth_type: 'user_pass', username: 'user', password: 'pass', - topics: [{ topic: 'topic1' }], + topic: 'topic1', is_default: true, }) .expect(200); @@ -1160,7 +1160,7 @@ export default function (providerContext: FtrProviderContext) { auth_type: 'user_pass', username: 'user', password: 'pass', - topics: [{ topic: 'topic1' }], + topic: 'topic1', is_default: true, is_default_monitoring: false, broker_timeout: 10, @@ -1388,7 +1388,7 @@ export default function (providerContext: FtrProviderContext) { auth_type: 'user_pass', username: 'user', password: 'pass', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1448,7 +1448,7 @@ export default function (providerContext: FtrProviderContext) { auth_type: 'user_pass', username: 'user', password: 'pass', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1469,7 +1469,7 @@ export default function (providerContext: FtrProviderContext) { type: 'kafka', hosts: ['test.fr:2000'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1501,7 +1501,7 @@ export default function (providerContext: FtrProviderContext) { type: 'kafka', hosts: ['test.fr:2000'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1536,7 +1536,7 @@ export default function (providerContext: FtrProviderContext) { hosts: ['test.fr:2000'], auth_type: 'user_pass', username: 'user', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1811,7 +1811,7 @@ export default function (providerContext: FtrProviderContext) { username: 'user', password: 'pass', is_default: true, - topics: [{ topic: 'topic1' }], + topic: 'topic1', }; before(async () => { @@ -1858,7 +1858,7 @@ export default function (providerContext: FtrProviderContext) { type: 'kafka', hosts: ['test.fr:2000'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, @@ -1905,7 +1905,7 @@ export default function (providerContext: FtrProviderContext) { type: 'kafka', hosts: ['test.fr:2000'], auth_type: 'ssl', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: '', compression: 'none', client_id: 'Elastic', @@ -1943,7 +1943,7 @@ export default function (providerContext: FtrProviderContext) { .expect(200); expect(updateRes.body.item.type).to.eql('logstash'); - expect(updateRes.body.item.topics).to.eql(null); + expect(updateRes.body.item.topic).to.eql(null); await supertest .delete(`/api/fleet/outputs/${outputWithSecretsId}`) diff --git a/x-pack/test/fleet_api_integration/apis/policy_secrets.ts b/x-pack/test/fleet_api_integration/apis/policy_secrets.ts index ce08126bf29d3..0f7215e9d204b 100644 --- a/x-pack/test/fleet_api_integration/apis/policy_secrets.ts +++ b/x-pack/test/fleet_api_integration/apis/policy_secrets.ts @@ -231,7 +231,7 @@ export default function (providerContext: FtrProviderContext) { hosts: ['test.fr:2000'], auth_type: 'user_pass', username: 'user', - topics: [{ topic: 'topic1' }], + topic: 'topic1', config_yaml: 'shipper: {}', shipper: { disk_queue_enabled: true, From e9e6ac9022a22f1fb2eb8bb3639a27fd5e5b227f Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 7 Nov 2024 09:02:49 -0500 Subject: [PATCH 7/7] fix cypress test --- x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts b/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts index 763e6fae1b1ef..5da321c52ba5a 100644 --- a/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts +++ b/x-pack/plugins/fleet/cypress/screens/fleet_outputs.ts @@ -66,7 +66,7 @@ export const kafkaOutputBody = { type: 'kafka', is_default: false, hosts: ['example.com:2000'], - topics: [{ topic: 'test' }], + topic: 'test', auth_type: 'user_pass', username: 'kafka', password: 'kafka',