diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts index dc98c34b20a39..f73015b576081 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts @@ -1069,6 +1069,22 @@ describe('Alerts Client', () => { }, }, }, + { + index: { + _index: '.internal.alerts-test.alerts-default-000001', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, { index: { _index: '.internal.alerts-test.alerts-default-000002', @@ -1106,7 +1122,7 @@ describe('Alerts Client', () => { expect(clusterClient.bulk).toHaveBeenCalled(); expect(logger.error).toHaveBeenCalledWith( - `Error writing alerts: 1 successful, 0 conflicts, 1 errors: Validation Failed: 1: index is missing;2: type is missing;` + `Error writing alerts: 1 successful, 0 conflicts, 2 errors: Validation Failed: 1: index is missing;2: type is missing;; failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.` ); }); diff --git a/x-pack/plugins/alerting/server/alerts_client/index.ts b/x-pack/plugins/alerting/server/alerts_client/index.ts index 442f8935650f5..a1c0a309e0dc4 100644 --- a/x-pack/plugins/alerting/server/alerts_client/index.ts +++ b/x-pack/plugins/alerting/server/alerts_client/index.ts @@ -8,3 +8,4 @@ export { type LegacyAlertsClientParams, LegacyAlertsClient } from './legacy_alerts_client'; export { AlertsClient } from './alerts_client'; export type { AlertRuleData } from './types'; +export { sanitizeBulkErrorResponse } from './lib'; diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts index 3c5ce6e25a1a8..383d8dbb103fb 100644 --- a/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts +++ b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts @@ -23,6 +23,7 @@ import { } from '@kbn/rule-data-utils'; import { zip, get } from 'lodash'; +import { sanitizeBulkErrorResponse } from '../..'; // these fields are the one's we'll refresh from the fresh mget'd docs const REFRESH_FIELDS_ALWAYS = [ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_TAGS, ALERT_CASE_IDS]; @@ -269,8 +270,9 @@ interface ResponseStatsResult { // generate a summary of the original bulk request attempt, for logging function getResponseStats(bulkResponse: BulkResponse): ResponseStatsResult { + const sanitizedResponse = sanitizeBulkErrorResponse(bulkResponse) as BulkResponse; const stats: ResponseStatsResult = { success: 0, conflicts: 0, errors: 0, messages: [] }; - for (const item of bulkResponse.items) { + for (const item of sanitizedResponse.items) { const op = item.create || item.index || item.update || item.delete; if (op?.error) { if (op?.status === 409 && op === item.index) { diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/index.ts b/x-pack/plugins/alerting/server/alerts_client/lib/index.ts index 7225e87056e4f..6e40e918a8b2c 100644 --- a/x-pack/plugins/alerting/server/alerts_client/lib/index.ts +++ b/x-pack/plugins/alerting/server/alerts_client/lib/index.ts @@ -16,3 +16,4 @@ export { getContinualAlertsQuery, } from './get_summarized_alerts_query'; export { expandFlattenedAlert } from './format_alert'; +export { sanitizeBulkErrorResponse } from './sanitize_bulk_response'; diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts new file mode 100644 index 0000000000000..533bb5b554ae9 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { TransportResult } from '@elastic/elasticsearch'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { sanitizeBulkErrorResponse } from './sanitize_bulk_response'; + +// Using https://www.elastic.co/guide/en/elasticsearch/reference/8.11/docs-bulk.html +describe('sanitizeBulkErrorResponse', () => { + test('should not modify success response', () => { + const responseBody = { + errors: false, + took: 1, + items: [ + { + index: { + _index: 'test', + _id: '1', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 0, + _primary_term: 1, + }, + }, + { + delete: { + _index: 'test', + _id: '2', + _version: 1, + result: 'not_found', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 404, + _seq_no: 1, + _primary_term: 2, + }, + }, + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + update: { + _index: 'test', + _id: '1', + _version: 2, + result: 'updated', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 200, + _seq_no: 3, + _primary_term: 4, + }, + }, + ], + }; + const transportResponseBody = wrapResponseBody(responseBody); + + expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody); + expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody); + }); + + test('should not modify error response without field preview', () => { + const responseBody = { + took: 486, + errors: true, + items: [ + { + update: { + _index: 'index1', + _id: '5', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[5]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + delete: { + _index: 'index1', + _id: '6', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[6]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + ], + }; + const transportResponseBody = wrapResponseBody(responseBody); + + expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody); + expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody); + }); + + test('should sanitize error response with field preview', () => { + const responseBody = { + took: 486, + errors: true, + items: [ + { + update: { + _index: 'index1', + _id: '5', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[5]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + update: { + _index: 'index1', + _id: '6', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[6]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }; + const transportResponseBody = wrapResponseBody(responseBody); + + expect(sanitizeBulkErrorResponse(responseBody)).toEqual({ + ...responseBody, + items: [ + responseBody.items[0], + responseBody.items[1], + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }); + expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual({ + ...transportResponseBody, + body: { + ...transportResponseBody.body, + items: [ + transportResponseBody.body.items[0], + transportResponseBody.body.items[1], + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }, + }); + }); +}); + +function wrapResponseBody( + body: estypes.BulkResponse, + statusCode: number = 200 +): TransportResult { + return { + body, + statusCode, + headers: {}, + warnings: null, + // @ts-expect-error + meta: {}, + }; +} diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts new file mode 100644 index 0000000000000..2b6d9f6e3c2c3 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { cloneDeep } from 'lodash'; +import { TransportResult } from '@elastic/elasticsearch'; +import { get } from 'lodash'; +import { set } from '@kbn/safer-lodash-set'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export const sanitizeBulkErrorResponse = ( + response: TransportResult | estypes.BulkResponse +): TransportResult | estypes.BulkResponse => { + const clonedResponse = cloneDeep(response); + const isTransportResponse = !!(response as TransportResult).body; + + const responseToUse: estypes.BulkResponse = isTransportResponse + ? (clonedResponse as TransportResult).body + : (clonedResponse as estypes.BulkResponse); + + if (responseToUse.errors) { + (responseToUse.items ?? []).forEach( + (item: Partial>) => { + for (const [_, responseItem] of Object.entries(item)) { + const reason: string = get(responseItem, 'error.reason'); + const redactIndex = reason ? reason.indexOf(`Preview of field's value:`) : -1; + if (redactIndex > 1) { + set(responseItem, 'error.reason', reason.substring(0, redactIndex - 1)); + } + } + } + ); + } + + return clonedResponse; +}; diff --git a/x-pack/plugins/alerting/server/index.ts b/x-pack/plugins/alerting/server/index.ts index 6aa1c44fe6e81..c2a800db66b51 100644 --- a/x-pack/plugins/alerting/server/index.ts +++ b/x-pack/plugins/alerting/server/index.ts @@ -67,6 +67,7 @@ export { installWithTimeout, isValidAlertIndexName, } from './alerts_service'; +export { sanitizeBulkErrorResponse } from './alerts_client'; export { getDataStreamAdapter } from './alerts_service/lib/data_stream_adapter'; export const plugin = (initContext: PluginInitializerContext) => new AlertingPlugin(initContext); diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts index 968a7fbadac0b..8898b8634b293 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts @@ -6,6 +6,7 @@ */ import { left, right } from 'fp-ts/lib/Either'; +import { errors } from '@elastic/elasticsearch'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { RuleDataClient, RuleDataClientConstructorOptions, WaitResult } from './rule_data_client'; @@ -382,6 +383,128 @@ describe('RuleDataClient', () => { expect(ruleDataClient.isWriteEnabled()).toBe(true); }); + test('sanitizes error before logging', async () => { + scopedClusterClient.bulk.mockResponseOnce({ + took: 486, + errors: true, + items: [ + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'test', + _id: '4', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }); + const ruleDataClient = new RuleDataClient( + getRuleDataClientOptions({ isUsingDataStreams }) + ); + expect(ruleDataClient.isWriteEnabled()).toBe(true); + const writer = await ruleDataClient.getWriter(); + + // Previously, a delay between calling getWriter() and using a writer function + // would cause an Unhandled promise rejection if there were any errors getting a writer + // Adding this delay in the tests to ensure this does not pop up again. + await delay(); + + const bulkWriteResponse = await writer.bulk({}); + expect(bulkWriteResponse).toEqual({ + body: { + took: 486, + errors: true, + items: [ + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'test', + _id: '4', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }, + headers: { + 'x-elastic-product': 'Elasticsearch', + }, + meta: {}, + statusCode: 200, + warnings: [], + }); + + expect(logger.error).toHaveBeenNthCalledWith( + 1, + // @ts-expect-error + new errors.ResponseError(bulkWriteResponse) + ); + expect(ruleDataClient.isWriteEnabled()).toBe(true); + }); + test('waits until cluster client is ready before calling bulk', async () => { scopedClusterClient.bulk.mockResolvedValueOnce( elasticsearchClientMock.createSuccessTransportRequestPromise( diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts index b4d029f4bbe82..329c060426093 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { errors } from '@elastic/elasticsearch'; +import { errors, TransportResult } from '@elastic/elasticsearch'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { Either, isLeft } from 'fp-ts/lib/Either'; @@ -14,6 +14,7 @@ import { Logger } from '@kbn/core/server'; import { IndexPatternsFetcher } from '@kbn/data-plugin/server'; import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; +import { sanitizeBulkErrorResponse } from '@kbn/alerting-plugin/server'; import { RuleDataWriteDisabledError, RuleDataWriterInitializationError, @@ -231,13 +232,20 @@ export class RuleDataClient implements IRuleDataClient { meta: true, }); + if (!response.body.errors) { + return response; + } + // TODO: #160572 - add support for version conflict errors, in case alert was updated // some other way between the time it was fetched and the time it was updated. - if (response.body.errors) { - const error = new errors.ResponseError(response); - this.options.logger.error(error); - } - return response; + // Redact part of reason message that echoes back value + const sanitizedResponse = sanitizeBulkErrorResponse(response) as TransportResult< + estypes.BulkResponse, + unknown + >; + const error = new errors.ResponseError(sanitizedResponse); + this.options.logger.error(error); + return sanitizedResponse; } else { this.options.logger.debug(`Writing is disabled, bulk() will not write any data.`); }