[8.11] [Response Ops][Alerting] Remove echoed field value from bulk e…

…rror responses when indexing alerts (#172020) (#172085)

# Backport

This will backport the following commits from `main` to `8.11`:
- [[Response Ops][Alerting] Remove echoed field value from bulk error
responses when indexing alerts
(#172020)](#172020)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Ying
Mao","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-11-28T16:25:03Z","message":"[Response
Ops][Alerting] Remove echoed field value from bulk error responses when
indexing alerts (#172020)\n\n## Summary\r\n\r\nWhen alerts are bulk
indexed in the rule registry and the alerts client,\r\nindexing errors
may be returned where the entire field value that failed\r\nto be
indexed is echoed in the reason. This can cause unnecessarily\r\nverbose
logging so we want to sanitize the field
value.","sha":"d9ebfd9af1365bba54d5e1ac92f5e53f5fbebea8","branchLabelMapping":{"^v8.12.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Feature:Alerting","release_note:skip","Team:ResponseOps","backport:prev-minor","backport:prev-MAJOR","v8.12.0"],"number":172020,"url":"https://github.com/elastic/kibana/pull/172020","mergeCommit":{"message":"[Response
Ops][Alerting] Remove echoed field value from bulk error responses when
indexing alerts (#172020)\n\n## Summary\r\n\r\nWhen alerts are bulk
indexed in the rule registry and the alerts client,\r\nindexing errors
may be returned where the entire field value that failed\r\nto be
indexed is echoed in the reason. This can cause unnecessarily\r\nverbose
logging so we want to sanitize the field
value.","sha":"d9ebfd9af1365bba54d5e1ac92f5e53f5fbebea8"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.12.0","labelRegex":"^v8.12.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/172020","number":172020,"mergeCommit":{"message":"[Response
Ops][Alerting] Remove echoed field value from bulk error responses when
indexing alerts (#172020)\n\n## Summary\r\n\r\nWhen alerts are bulk
indexed in the rule registry and the alerts client,\r\nindexing errors
may be returned where the entire field value that failed\r\nto be
indexed is echoed in the reason. This can cause unnecessarily\r\nverbose
logging so we want to sanitize the field
value.","sha":"d9ebfd9af1365bba54d5e1ac92f5e53f5fbebea8"}}]}]
BACKPORT-->

Co-authored-by: Ying Mao <[email protected]>

x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts

@@ -1069,6 +1069,22 @@ describe('Alerts Client', () => {
                   },
                 },
               },
+              {
+                index: {
+                  _index: '.internal.alerts-test.alerts-default-000001',
+                  _id: '7',
+                  status: 404,
+                  error: {
+                    type: 'mapper_parsing_exception',
+                    reason:
+                      "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'",
+                    caused_by: {
+                      type: 'illegal_state_exception',
+                      reason: "Can't get text on a START_OBJECT at 1:3845",
+                    },
+                  },
+                },
+              },
               {
                 index: {
                   _index: '.internal.alerts-test.alerts-default-000002',
@@ -1106,7 +1122,7 @@ describe('Alerts Client', () => {
 
           expect(clusterClient.bulk).toHaveBeenCalled();
           expect(logger.error).toHaveBeenCalledWith(
-            `Error writing alerts: 1 successful, 0 conflicts, 1 errors: Validation Failed: 1: index is missing;2: type is missing;`
+            `Error writing alerts: 1 successful, 0 conflicts, 2 errors: Validation Failed: 1: index is missing;2: type is missing;; failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.`
           );
         });
 

x-pack/plugins/alerting/server/alerts_client/index.ts

@@ -8,3 +8,4 @@
 export { type LegacyAlertsClientParams, LegacyAlertsClient } from './legacy_alerts_client';
 export { AlertsClient } from './alerts_client';
 export type { AlertRuleData } from './types';
+export { sanitizeBulkErrorResponse } from './lib';

x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts

@@ -23,6 +23,7 @@ import {
 } from '@kbn/rule-data-utils';
 
 import { zip, get } from 'lodash';
+import { sanitizeBulkErrorResponse } from '../..';
 
 // these fields are the one's we'll refresh from the fresh mget'd docs
 const REFRESH_FIELDS_ALWAYS = [ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_TAGS, ALERT_CASE_IDS];
@@ -269,8 +270,9 @@ interface ResponseStatsResult {
 
 // generate a summary of the original bulk request attempt, for logging
 function getResponseStats(bulkResponse: BulkResponse): ResponseStatsResult {
+  const sanitizedResponse = sanitizeBulkErrorResponse(bulkResponse) as BulkResponse;
   const stats: ResponseStatsResult = { success: 0, conflicts: 0, errors: 0, messages: [] };
-  for (const item of bulkResponse.items) {
+  for (const item of sanitizedResponse.items) {
     const op = item.create || item.index || item.update || item.delete;
     if (op?.error) {
       if (op?.status === 409 && op === item.index) {

x-pack/plugins/alerting/server/alerts_client/lib/index.ts

@@ -16,3 +16,4 @@ export {
   getContinualAlertsQuery,
 } from './get_summarized_alerts_query';
 export { expandFlattenedAlert } from './format_alert';
+export { sanitizeBulkErrorResponse } from './sanitize_bulk_response';

x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts

@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { TransportResult } from '@elastic/elasticsearch';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { sanitizeBulkErrorResponse } from './sanitize_bulk_response';
+
+// Using https://www.elastic.co/guide/en/elasticsearch/reference/8.11/docs-bulk.html
+describe('sanitizeBulkErrorResponse', () => {
+  test('should not modify success response', () => {
+    const responseBody = {
+      errors: false,
+      took: 1,
+      items: [
+        {
+          index: {
+            _index: 'test',
+            _id: '1',
+            _version: 1,
+            result: 'created',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 201,
+            _seq_no: 0,
+            _primary_term: 1,
+          },
+        },
+        {
+          delete: {
+            _index: 'test',
+            _id: '2',
+            _version: 1,
+            result: 'not_found',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 404,
+            _seq_no: 1,
+            _primary_term: 2,
+          },
+        },
+        {
+          create: {
+            _index: 'test',
+            _id: '3',
+            _version: 1,
+            result: 'created',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 201,
+            _seq_no: 2,
+            _primary_term: 3,
+          },
+        },
+        {
+          update: {
+            _index: 'test',
+            _id: '1',
+            _version: 2,
+            result: 'updated',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 200,
+            _seq_no: 3,
+            _primary_term: 4,
+          },
+        },
+      ],
+    };
+    const transportResponseBody = wrapResponseBody(responseBody);
+
+    expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody);
+    expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody);
+  });
+
+  test('should not modify error response without field preview', () => {
+    const responseBody = {
+      took: 486,
+      errors: true,
+      items: [
+        {
+          update: {
+            _index: 'index1',
+            _id: '5',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[5]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          delete: {
+            _index: 'index1',
+            _id: '6',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[6]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          create: {
+            _index: 'test',
+            _id: '3',
+            _version: 1,
+            result: 'created',
+            _shards: { total: 2, successful: 1, failed: 0 },
+            status: 201,
+            _seq_no: 2,
+            _primary_term: 3,
+          },
+        },
+      ],
+    };
+    const transportResponseBody = wrapResponseBody(responseBody);
+
+    expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody);
+    expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody);
+  });
+
+  test('should sanitize error response with field preview', () => {
+    const responseBody = {
+      took: 486,
+      errors: true,
+      items: [
+        {
+          update: {
+            _index: 'index1',
+            _id: '5',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[5]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          update: {
+            _index: 'index1',
+            _id: '6',
+            status: 404,
+            error: {
+              type: 'document_missing_exception',
+              reason: '[6]: document missing',
+              index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA',
+              shard: '0',
+              index: 'index1',
+            },
+          },
+        },
+        {
+          create: {
+            _index: 'index1',
+            _id: '7',
+            status: 404,
+            error: {
+              type: 'mapper_parsing_exception',
+              reason:
+                "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'",
+              caused_by: {
+                type: 'illegal_state_exception',
+                reason: "Can't get text on a START_OBJECT at 1:3845",
+              },
+            },
+          },
+        },
+      ],
+    };
+    const transportResponseBody = wrapResponseBody(responseBody);
+
+    expect(sanitizeBulkErrorResponse(responseBody)).toEqual({
+      ...responseBody,
+      items: [
+        responseBody.items[0],
+        responseBody.items[1],
+        {
+          create: {
+            _index: 'index1',
+            _id: '7',
+            status: 404,
+            error: {
+              type: 'mapper_parsing_exception',
+              reason:
+                "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.",
+              caused_by: {
+                type: 'illegal_state_exception',
+                reason: "Can't get text on a START_OBJECT at 1:3845",
+              },
+            },
+          },
+        },
+      ],
+    });
+    expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual({
+      ...transportResponseBody,
+      body: {
+        ...transportResponseBody.body,
+        items: [
+          transportResponseBody.body.items[0],
+          transportResponseBody.body.items[1],
+          {
+            create: {
+              _index: 'index1',
+              _id: '7',
+              status: 404,
+              error: {
+                type: 'mapper_parsing_exception',
+                reason:
+                  "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.",
+                caused_by: {
+                  type: 'illegal_state_exception',
+                  reason: "Can't get text on a START_OBJECT at 1:3845",
+                },
+              },
+            },
+          },
+        ],
+      },
+    });
+  });
+});
+
+function wrapResponseBody(
+  body: estypes.BulkResponse,
+  statusCode: number = 200
+): TransportResult<estypes.BulkResponse, unknown> {
+  return {
+    body,
+    statusCode,
+    headers: {},
+    warnings: null,
+    // @ts-expect-error
+    meta: {},
+  };
+}

x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { cloneDeep } from 'lodash';
+import { TransportResult } from '@elastic/elasticsearch';
+import { get } from 'lodash';
+import { set } from '@kbn/safer-lodash-set';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+export const sanitizeBulkErrorResponse = (
+  response: TransportResult<estypes.BulkResponse, unknown> | estypes.BulkResponse
+): TransportResult<estypes.BulkResponse, unknown> | estypes.BulkResponse => {
+  const clonedResponse = cloneDeep(response);
+  const isTransportResponse = !!(response as TransportResult<estypes.BulkResponse, unknown>).body;
+
+  const responseToUse: estypes.BulkResponse = isTransportResponse
+    ? (clonedResponse as TransportResult<estypes.BulkResponse, unknown>).body
+    : (clonedResponse as estypes.BulkResponse);
+
+  if (responseToUse.errors) {
+    (responseToUse.items ?? []).forEach(
+      (item: Partial<Record<estypes.BulkOperationType, estypes.BulkResponseItem>>) => {
+        for (const [_, responseItem] of Object.entries(item)) {
+          const reason: string = get(responseItem, 'error.reason');
+          const redactIndex = reason ? reason.indexOf(`Preview of field's value:`) : -1;
+          if (redactIndex > 1) {
+            set(responseItem, 'error.reason', reason.substring(0, redactIndex - 1));
+          }
+        }
+      }
+    );
+  }
+
+  return clonedResponse;
+};

x-pack/plugins/alerting/server/index.ts

@@ -67,6 +67,7 @@ export {
   installWithTimeout,
   isValidAlertIndexName,
 } from './alerts_service';
+export { sanitizeBulkErrorResponse } from './alerts_client';
 export { getDataStreamAdapter } from './alerts_service/lib/data_stream_adapter';
 
 export const plugin = (initContext: PluginInitializerContext) => new AlertingPlugin(initContext);