From 6e87efd01c8db3815248e060f96bac43909fa110 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 14 Nov 2024 21:41:18 +0100 Subject: [PATCH 1/3] place nice with ES --- .../data_streams/manage_data_streams.ts | 51 ++++++++++--------- .../streams/server/lib/streams/stream_crud.ts | 37 ++++++++++++-- .../streams/server/routes/streams/list.ts | 28 +++++----- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts index 812739db56c73..a9b667906fdf3 100644 --- a/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts +++ b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -6,6 +6,7 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import { retryTransientEsErrors } from '../helpers/retry'; interface DataStreamManagementOptions { @@ -23,6 +24,7 @@ interface DeleteDataStreamOptions { interface RolloverDataStreamOptions { esClient: ElasticsearchClient; name: string; + mappings: MappingTypeMapping['properties'] | undefined; logger: Logger; } @@ -56,38 +58,37 @@ export async function rolloverDataStreamIfNecessary({ esClient, name, logger, + mappings, }: RolloverDataStreamOptions) { const dataStreams = await esClient.indices.getDataStream({ name: `${name},${name}.*` }); for (const dataStream of dataStreams.data_streams) { - const currentMappings = - Object.values( - await esClient.indices.getMapping({ - index: dataStream.indices.at(-1)?.index_name, - }) - )[0].mappings.properties || {}; - const simulatedIndex = await esClient.indices.simulateIndexTemplate({ name: dataStream.name }); - const simulatedMappings = simulatedIndex.template.mappings.properties || {}; - - // check whether the same fields and same types are listed (don't check for other mapping attributes) - const isDifferent = - Object.values(simulatedMappings).length !== Object.values(currentMappings).length || - Object.entries(simulatedMappings || {}).some(([fieldName, { type }]) => { - const currentType = currentMappings[fieldName]?.type; - return currentType !== type; - }); - - if (!isDifferent) { + const writeIndex = dataStream.indices.at(-1); + if (!writeIndex) { continue; } - try { - await retryTransientEsErrors(() => esClient.indices.rollover({ alias: dataStream.name }), { - logger, - }); - logger.debug(() => `Rolled over data stream: ${dataStream.name}`); + await retryTransientEsErrors( + () => esClient.indices.putMapping({ index: writeIndex.index_name, properties: mappings }), + { + logger, + } + ); } catch (error: any) { - logger.error(`Error rolling over data stream: ${error.message}`); - throw error; + if ( + typeof error.message !== 'string' || + !error.message.includes('illegal_argument_exception') + ) { + throw error; + } + try { + await retryTransientEsErrors(() => esClient.indices.rollover({ alias: dataStream.name }), { + logger, + }); + logger.debug(() => `Rolled over data stream: ${dataStream.name}`); + } catch (rolloverError: any) { + logger.error(`Error rolling over data stream: ${error.message}`); + throw error; + } } } } diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 78a126905d9a4..c85d37df8267d 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -97,20 +97,30 @@ export async function listStreams({ scopedClusterClient }: ListStreamsParams) { sort: [{ id: 'asc' }], }); const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] }); - return definitions; + const hasAccess = await Promise.all( + definitions.map((definition) => checkReadAccess({ id: definition.id[0], scopedClusterClient })) + ); + return definitions.filter((_, index) => hasAccess[index]); } interface ReadStreamParams extends BaseParams { id: string; + skipAccessCheck?: boolean; } -export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { +export async function readStream({ id, scopedClusterClient, skipAccessCheck }: ReadStreamParams) { try { const response = await scopedClusterClient.asInternalUser.get({ id, index: STREAMS_INDEX, }); const definition = response._source as StreamDefinition; + if (!skipAccessCheck) { + const hasAccess = await checkReadAccess({ id, scopedClusterClient }); + if (!hasAccess) { + throw new DefinitionNotFound(`Stream definition for ${id} not found.`); + } + } return { definition, }; @@ -130,7 +140,9 @@ export async function readAncestors({ id, scopedClusterClient }: ReadAncestorsPa const ancestorIds = getAncestors(id); return await Promise.all( - ancestorIds.map((ancestorId) => readStream({ scopedClusterClient, id: ancestorId })) + ancestorIds.map((ancestorId) => + readStream({ scopedClusterClient, id: ancestorId, skipAccessCheck: true }) + ) ); } @@ -223,6 +235,21 @@ export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamP } } +interface CheckReadAccessParams extends BaseParams { + id: string; +} + +export async function checkReadAccess({ + id, + scopedClusterClient, +}: CheckReadAccessParams): Promise { + try { + return await scopedClusterClient.asCurrentUser.indices.exists({ index: id }); + } catch (e) { + return false; + } +} + interface SyncStreamParams { scopedClusterClient: IScopedClusterClient; definition: StreamDefinition; @@ -236,10 +263,11 @@ export async function syncStream({ rootDefinition, logger, }: SyncStreamParams) { + const componentTemplate = generateLayer(definition.id, definition); await upsertComponent({ esClient: scopedClusterClient.asCurrentUser, logger, - component: generateLayer(definition.id, definition), + component: componentTemplate, }); await upsertIngestPipeline({ esClient: scopedClusterClient.asCurrentUser, @@ -282,5 +310,6 @@ export async function syncStream({ esClient: scopedClusterClient.asCurrentUser, name: definition.id, logger, + mappings: componentTemplate.template.mappings?.properties, }); } diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index 2e4f13a89bb41..6de57b4d25581 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -51,20 +51,22 @@ interface ListStreamDefinition { function asTrees(definitions: Array<{ id: string[] }>) { const trees: ListStreamDefinition[] = []; - definitions.forEach((definition) => { - const path = definition.id[0].split('.'); + const ids = definitions.map((definition) => definition.id[0]); + + ids.sort((a, b) => a.split('.').length - b.split('.').length); + + ids.forEach((id) => { let currentTree = trees; - path.forEach((_id, index) => { - const partialPath = path.slice(0, index + 1).join('.'); - const existingNode = currentTree.find((node) => node.id === partialPath); - if (existingNode) { - currentTree = existingNode.children; - } else { - const newNode = { id: partialPath, children: [] }; - currentTree.push(newNode); - currentTree = newNode.children; - } - }); + let existingNode: ListStreamDefinition | undefined; + // traverse the tree following the prefix of the current id. + // once we reach the leaf, the current id is added as child - this works because the ids are sorted by depth + while ((existingNode = currentTree.find((node) => id.startsWith(node.id)))) { + currentTree = existingNode.children; + } + if (!existingNode) { + const newNode = { id, children: [] }; + currentTree.push(newNode); + } }); return trees; } From d0c05f337974b3fef5e132ed5ce12538487b805a Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 14 Nov 2024 21:00:30 +0000 Subject: [PATCH 2/3] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.serverless.yaml | 78 +++++---------------- oas_docs/output/kibana.yaml | 96 ++++++-------------------- 2 files changed, 40 insertions(+), 134 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 117e52586c5ad..4f54e401b14c2 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -1018,24 +1018,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -1610,24 +1603,17 @@ paths: type: boolean flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -1945,24 +1931,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2540,24 +2519,17 @@ paths: - active flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2847,24 +2819,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -3902,24 +3867,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index ceefaa13fcd4b..cb7d39cae0cab 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -1367,24 +1367,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -1958,24 +1951,17 @@ paths: type: boolean flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2293,24 +2279,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2887,24 +2866,17 @@ paths: - active flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -3194,24 +3166,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -4241,24 +4206,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -6708,14 +6666,9 @@ paths: - cases /api/cases/{caseId}/files: post: - description: > - Attach a file to a case. You must have `all` privileges for the - **Cases** feature in the **Management**, **Observability**, or - **Security** section of the Kibana feature privileges, depending on the - owner of the case you're updating. The request must include: - + description: | + Attach a file to a case. You must have `all` privileges for the **Cases** feature in the **Management**, **Observability**, or **Security** section of the Kibana feature privileges, depending on the owner of the case you're updating. The request must include: - The `Content-Type: multipart/form-data` HTTP header. - - The location of the file that is being uploaded. operationId: addCaseFileDefaultSpace parameters: @@ -43715,9 +43668,7 @@ components: - $ref: '#/components/schemas/Cases_add_user_comment_request_properties' title: Add case comment request Cases_add_case_file_request: - description: >- - Defines the file that will be attached to the case. Optional parameters - will be generated automatically from the file metadata if not defined. + description: Defines the file that will be attached to the case. Optional parameters will be generated automatically from the file metadata if not defined. type: object properties: file: @@ -43725,10 +43676,7 @@ components: format: binary type: string filename: - description: >- - The desired name of the file being attached to the case, it can be - different than the name of the file in the filesystem. **This should - not include the file extension.** + description: The desired name of the file being attached to the case, it can be different than the name of the file in the filesystem. **This should not include the file extension.** type: string required: - file From a55fe173b3b454b0ed04bd8f90d4b7dd41eaee72 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 25 Nov 2024 17:22:00 +0100 Subject: [PATCH 3/3] fix --- .../streams/server/routes/streams/list.ts | 37 +++---------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index 23b1901dbfe47..d3b88ffc36a45 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -52,16 +52,15 @@ export interface StreamTree { children: StreamTree[]; } -<<<<<<< HEAD -function asTrees(definitions: Array<{ id: string[] }>) { - const trees: ListStreamDefinition[] = []; - const ids = definitions.map((definition) => definition.id[0]); +function asTrees(definitions: Array<{ id: string }>) { + const trees: StreamTree[] = []; + const ids = definitions.map((definition) => definition.id); ids.sort((a, b) => a.split('.').length - b.split('.').length); ids.forEach((id) => { let currentTree = trees; - let existingNode: ListStreamDefinition | undefined; + let existingNode: StreamTree | undefined; // traverse the tree following the prefix of the current id. // once we reach the leaf, the current id is added as child - this works because the ids are sorted by depth while ((existingNode = currentTree.find((node) => id.startsWith(node.id)))) { @@ -70,34 +69,8 @@ function asTrees(definitions: Array<{ id: string[] }>) { if (!existingNode) { const newNode = { id, children: [] }; currentTree.push(newNode); -======= -function asTrees(definitions: StreamDefinition[]): StreamTree[] { - const nodes = new Map(); - - const rootNodes = new Set(); - - function getNode(id: string) { - let node = nodes.get(id); - if (!node) { - node = { id, children: [] }; - nodes.set(id, node); - } - return node; - } - - definitions.forEach((definition) => { - const path = definition.id.split('.'); - const parentId = path.slice(0, path.length - 1).join('.'); - const parentNode = parentId.length ? getNode(parentId) : undefined; - const selfNode = getNode(definition.id); - - if (parentNode) { - parentNode.children.push(selfNode); - } else { - rootNodes.add(selfNode); ->>>>>>> upstream/main } }); - return Array.from(rootNodes.values()); + return trees; }