From 63535d45580071ff7b83d69c5c66d8a16c50fcbc Mon Sep 17 00:00:00 2001
From: Karen Grigoryan <karen.grigoryan@elastic.co>
Date: Mon, 4 Nov 2024 16:09:12 +0100
Subject: [PATCH] [Security Solution][Data Quality Dashboard] fix pattern state
 reset on ilm phase filter change (#198549)

addresses #196523

- Fixes ilm phase change propagation on patterns.
- Adds missing tests for useResultsRollup functionality

## UI changes

### Before

https://github.com/user-attachments/assets/78a1d809-6a9a-4bfc-88a9-079f829a2017

### After

https://github.com/user-attachments/assets/f689fcc9-e1c6-4ccf-a7ca-8f13e9507ba4
(cherry picked from commit ddf55ea3a79ad7439dc02cb1f93291b1bc95c3b9)
---
 .../use_stored_pattern_results/index.test.tsx | 108 +++
 .../use_stored_pattern_results/index.tsx      |  53 ++
 .../hooks/use_results_rollup/index.test.tsx   | 685 ++++++++++++++++++
 .../hooks/use_results_rollup/index.tsx        |  66 +-
 .../impl/data_quality_panel/index.tsx         |   1 -
 .../mock/test_providers/utils/format.ts       |  17 +
 .../get_merged_data_quality_context_props.ts  |  11 +-
 .../stub/get_pattern_rollup_stub/index.ts     | 116 +++
 8 files changed, 986 insertions(+), 71 deletions(-)
 create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx
 create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx
 create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx
 create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/format.ts
 create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stub/get_pattern_rollup_stub/index.ts

diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx
new file mode 100644
index 0000000000000..d58bf3af39d58
--- /dev/null
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { renderHook } from '@testing-library/react-hooks';
+import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
+
+import { getHistoricalResultStub } from '../../../../stub/get_historical_result_stub';
+import { useStoredPatternResults } from '.';
+
+describe('useStoredPatternResults', () => {
+  const httpFetch = jest.fn();
+  const mockToasts = notificationServiceMock.createStartContract().toasts;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('when patterns are empty', () => {
+    it('should return an empty array and not call getStorageResults', () => {
+      const { result } = renderHook(() => useStoredPatternResults([], mockToasts, httpFetch));
+
+      expect(result.current).toEqual([]);
+      expect(httpFetch).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('when patterns are provided', () => {
+    it('should fetch and return stored pattern results correctly', async () => {
+      const patterns = ['pattern1-*', 'pattern2-*'];
+
+      httpFetch.mockImplementation((path: string) => {
+        if (path === '/internal/ecs_data_quality_dashboard/results_latest/pattern1-*') {
+          return Promise.resolve([getHistoricalResultStub('pattern1-index1')]);
+        }
+
+        if (path === '/internal/ecs_data_quality_dashboard/results_latest/pattern2-*') {
+          return Promise.resolve([getHistoricalResultStub('pattern2-index1')]);
+        }
+
+        return Promise.reject(new Error('Invalid path'));
+      });
+
+      const { result, waitFor } = renderHook(() =>
+        useStoredPatternResults(patterns, mockToasts, httpFetch)
+      );
+
+      await waitFor(() => result.current.length > 0);
+
+      expect(httpFetch).toHaveBeenCalledTimes(2);
+
+      expect(httpFetch).toHaveBeenCalledWith(
+        '/internal/ecs_data_quality_dashboard/results_latest/pattern1-*',
+        {
+          method: 'GET',
+          signal: expect.any(AbortSignal),
+          version: '1',
+        }
+      );
+      expect(httpFetch).toHaveBeenCalledWith(
+        '/internal/ecs_data_quality_dashboard/results_latest/pattern2-*',
+        {
+          method: 'GET',
+          signal: expect.any(AbortSignal),
+          version: '1',
+        }
+      );
+
+      expect(result.current).toEqual([
+        {
+          pattern: 'pattern1-*',
+          results: {
+            'pattern1-index1': {
+              docsCount: expect.any(Number),
+              error: null,
+              ilmPhase: expect.any(String),
+              incompatible: expect.any(Number),
+              indexName: 'pattern1-index1',
+              pattern: 'pattern1-*',
+              markdownComments: expect.any(Array),
+              sameFamily: expect.any(Number),
+              checkedAt: expect.any(Number),
+            },
+          },
+        },
+        {
+          pattern: 'pattern2-*',
+          results: {
+            'pattern2-index1': {
+              docsCount: expect.any(Number),
+              error: null,
+              ilmPhase: expect.any(String),
+              incompatible: expect.any(Number),
+              indexName: 'pattern2-index1',
+              pattern: 'pattern2-*',
+              markdownComments: expect.any(Array),
+              sameFamily: expect.any(Number),
+              checkedAt: expect.any(Number),
+            },
+          },
+        },
+      ]);
+    });
+  });
+});
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx
new file mode 100644
index 0000000000000..17334c4b4a586
--- /dev/null
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { useEffect, useState } from 'react';
+import { IToasts } from '@kbn/core-notifications-browser';
+import { HttpHandler } from '@kbn/core-http-browser';
+import { isEmpty } from 'lodash/fp';
+
+import { DataQualityCheckResult } from '../../../../types';
+import { formatResultFromStorage, getStorageResults } from '../../utils/storage';
+
+export const useStoredPatternResults = (
+  patterns: string[],
+  toasts: IToasts,
+  httpFetch: HttpHandler
+) => {
+  const [storedPatternResults, setStoredPatternResults] = useState<
+    Array<{ pattern: string; results: Record<string, DataQualityCheckResult> }>
+  >([]);
+
+  useEffect(() => {
+    if (isEmpty(patterns)) {
+      return;
+    }
+
+    const abortController = new AbortController();
+    const fetchStoredPatternResults = async () => {
+      const requests = patterns.map((pattern) =>
+        getStorageResults({ pattern, httpFetch, abortController, toasts }).then((results = []) => ({
+          pattern,
+          results: Object.fromEntries(
+            results.map((storageResult) => [
+              storageResult.indexName,
+              formatResultFromStorage({ storageResult, pattern }),
+            ])
+          ),
+        }))
+      );
+
+      const patternResults = await Promise.all(requests);
+      if (patternResults?.length) {
+        setStoredPatternResults(patternResults);
+      }
+    };
+
+    fetchStoredPatternResults();
+  }, [httpFetch, patterns, toasts]);
+
+  return storedPatternResults;
+};
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx
new file mode 100644
index 0000000000000..bff3c3dd54f12
--- /dev/null
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx
@@ -0,0 +1,685 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// fixing timezone for Date
+// so when tests are run in different timezones, the results are consistent
+process.env.TZ = 'UTC';
+
+import { renderHook, act } from '@testing-library/react-hooks';
+import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
+
+import type { TelemetryEvents } from '../../types';
+import { useStoredPatternResults } from './hooks/use_stored_pattern_results';
+import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
+import { useResultsRollup } from '.';
+import { getPatternRollupStub } from '../../stub/get_pattern_rollup_stub';
+import { formatBytes, formatNumber } from '../../mock/test_providers/utils/format';
+
+jest.mock('./hooks/use_stored_pattern_results', () => ({
+  ...jest.requireActual('./hooks/use_stored_pattern_results'),
+  useStoredPatternResults: jest.fn().mockReturnValue([]),
+}));
+
+describe('useResultsRollup', () => {
+  const httpFetch = jest.fn();
+  const toasts = notificationServiceMock.createStartContract().toasts;
+
+  const mockTelemetryEvents: TelemetryEvents = {
+    reportDataQualityIndexChecked: jest.fn(),
+    reportDataQualityCheckAllCompleted: jest.fn(),
+  };
+
+  const patterns = ['auditbeat-*', 'packetbeat-*'];
+  const isILMAvailable = true;
+
+  const useStoredPatternResultsMock = useStoredPatternResults as jest.Mock;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    useStoredPatternResultsMock.mockReturnValue([]);
+  });
+
+  describe('initialization', () => {
+    it('should initialize with default values', () => {
+      const { result } = renderHook(() =>
+        useResultsRollup({
+          httpFetch,
+          toasts,
+          patterns,
+          isILMAvailable,
+          telemetryEvents: mockTelemetryEvents,
+        })
+      );
+
+      expect(result.current.patternIndexNames).toEqual({});
+      expect(result.current.patternRollups).toEqual({});
+      expect(result.current.totalDocsCount).toBe(0);
+      expect(result.current.totalIncompatible).toBeUndefined();
+      expect(result.current.totalIndices).toBe(0);
+      expect(result.current.totalIndicesChecked).toBe(0);
+      expect(result.current.totalSameFamily).toBeUndefined();
+      expect(result.current.totalSizeInBytes).toBe(0);
+    });
+
+    it('should fetch stored pattern results and update patternRollups from it', () => {
+      const mockStoredResults = [
+        {
+          pattern: 'auditbeat-*',
+          results: {
+            'auditbeat-7.11.0-2021.01.01': {
+              indexName: 'auditbeat-7.11.0-2021.01.01',
+              pattern: 'auditbeat-*',
+              docsCount: 500,
+              incompatible: 0,
+              error: null,
+              ilmPhase: 'hot',
+              sameFamily: 0,
+              markdownComments: [],
+              checkedAt: Date.now(),
+            },
+          },
+        },
+      ];
+
+      useStoredPatternResultsMock.mockReturnValue(mockStoredResults);
+
+      const { result } = renderHook(() =>
+        useResultsRollup({
+          httpFetch,
+          toasts,
+          patterns: ['auditbeat-*'],
+          isILMAvailable,
+          telemetryEvents: mockTelemetryEvents,
+        })
+      );
+
+      expect(useStoredPatternResultsMock).toHaveBeenCalledWith(['auditbeat-*'], toasts, httpFetch);
+
+      expect(result.current.patternRollups).toEqual({
+        'auditbeat-*': {
+          pattern: 'auditbeat-*',
+          results: {
+            'auditbeat-7.11.0-2021.01.01': expect.any(Object),
+          },
+        },
+      });
+    });
+  });
+
+  describe('updatePatternIndexNames', () => {
+    it('should update pattern index names', () => {
+      const { result } = renderHook(() =>
+        useResultsRollup({
+          httpFetch,
+          toasts,
+          patterns,
+          isILMAvailable,
+          telemetryEvents: mockTelemetryEvents,
+        })
+      );
+
+      act(() => {
+        result.current.updatePatternIndexNames({
+          pattern: 'packetbeat-*',
+          indexNames: ['packetbeat-7.10.0-2021.01.01'],
+        });
+      });
+
+      expect(result.current.patternIndexNames).toEqual({
+        'packetbeat-*': ['packetbeat-7.10.0-2021.01.01'],
+      });
+    });
+  });
+
+  describe('updatePatternRollup', () => {
+    it('should update pattern rollup when called', () => {
+      const { result } = renderHook(() =>
+        useResultsRollup({
+          httpFetch,
+          toasts,
+          patterns,
+          isILMAvailable,
+          telemetryEvents: mockTelemetryEvents,
+        })
+      );
+
+      const patternRollup = getPatternRollupStub('packetbeat-*', 1);
+
+      expect(result.current.patternRollups).toEqual({});
+
+      act(() => {
+        result.current.updatePatternRollup(patternRollup);
+      });
+
+      expect(result.current.patternRollups).toEqual({
+        'packetbeat-*': patternRollup,
+      });
+    });
+  });
+
+  describe('onCheckCompleted', () => {
+    describe('when invoked with successful check data', () => {
+      beforeEach(() => {
+        jest.useFakeTimers();
+        jest.setSystemTime(new Date('2021-10-07T00:00:00Z').getTime());
+      });
+
+      afterEach(() => {
+        jest.useRealTimers();
+      });
+
+      it('should update patternRollup with said data, report to telemetry and persist it in storage', () => {
+        const { result } = renderHook(() =>
+          useResultsRollup({
+            httpFetch,
+            toasts,
+            patterns,
+            isILMAvailable,
+            telemetryEvents: mockTelemetryEvents,
+          })
+        );
+
+        const patternRollup = getPatternRollupStub('packetbeat-*', 1);
+
+        act(() => {
+          result.current.updatePatternRollup(patternRollup);
+        });
+
+        expect(result.current.patternRollups['packetbeat-*'].results?.['.ds-packetbeat-1']).toEqual(
+          {
+            checkedAt: new Date('2021-10-07T00:00:00Z').getTime(),
+            docsCount: 1000000,
+            error: null,
+            ilmPhase: 'hot',
+            incompatible: 0,
+            indexName: '.ds-packetbeat-1',
+            markdownComments: ['foo', 'bar', 'baz'],
+            pattern: 'packetbeat-*',
+            sameFamily: 0,
+          }
+        );
+
+        jest.advanceTimersByTime(1000);
+
+        const mockOnCheckCompletedOpts = {
+          batchId: 'test-batch',
+          checkAllStartTime: Date.now(),
+          error: null,
+          formatBytes,
+          formatNumber,
+          indexName: '.ds-packetbeat-1',
+          partitionedFieldMetadata: mockPartitionedFieldMetadata,
+          pattern: 'packetbeat-*',
+          requestTime: 1500,
+          isLastCheck: true,
+          isCheckAll: true,
+        };
+
+        jest.advanceTimersByTime(1000);
+
+        act(() => {
+          result.current.onCheckCompleted(mockOnCheckCompletedOpts);
+        });
+
+        expect(result.current.patternRollups['packetbeat-*'].results?.['.ds-packetbeat-1']).toEqual(
+          {
+            checkedAt: new Date('2021-10-07T00:00:02Z').getTime(),
+            docsCount: 1000000,
+            error: null,
+            ilmPhase: 'hot',
+            incompatible: 3,
+            indexName: '.ds-packetbeat-1',
+            markdownComments: expect.any(Array),
+            pattern: 'packetbeat-*',
+            sameFamily: 0,
+          }
+        );
+
+        expect(mockTelemetryEvents.reportDataQualityIndexChecked).toHaveBeenCalledWith({
+          batchId: 'test-batch',
+          ecsVersion: '8.11.0',
+          errorCount: 0,
+          ilmPhase: 'hot',
+          indexId: 'uuid-1',
+          indexName: '.ds-packetbeat-1',
+          isCheckAll: true,
+          numberOfCustomFields: 4,
+          numberOfDocuments: 1000000,
+          numberOfEcsFields: 2,
+          numberOfFields: 9,
+          numberOfIncompatibleFields: 3,
+          numberOfIndices: 1,
+          numberOfIndicesChecked: 1,
+          numberOfSameFamily: 0,
+          sameFamilyFields: [],
+          sizeInBytes: 500000000,
+          timeConsumedMs: 1500,
+          unallowedMappingFields: ['host.name', 'source.ip'],
+          unallowedValueFields: ['event.category'],
+        });
+        expect(mockTelemetryEvents.reportDataQualityCheckAllCompleted).toHaveBeenCalledWith({
+          batchId: 'test-batch',
+          ecsVersion: '8.11.0',
+          isCheckAll: true,
+          numberOfDocuments: 1000000,
+          numberOfIncompatibleFields: 3,
+          numberOfIndices: 1,
+          numberOfIndicesChecked: 1,
+          numberOfSameFamily: 0,
+          sizeInBytes: 500000000,
+          timeConsumedMs: 1000,
+        });
+
+        expect(httpFetch).toHaveBeenCalledWith('/internal/ecs_data_quality_dashboard/results', {
+          method: 'POST',
+          version: '1',
+          signal: expect.any(AbortSignal),
+          body: expect.any(String),
+        });
+
+        const body = JSON.parse(httpFetch.mock.calls[0][1].body);
+
+        expect(body).toEqual({
+          batchId: 'test-batch',
+          indexName: '.ds-packetbeat-1',
+          indexPattern: 'packetbeat-*',
+          isCheckAll: true,
+          checkedAt: new Date('2021-10-07T00:00:02Z').getTime(),
+          docsCount: 1000000,
+          totalFieldCount: 9,
+          ecsFieldCount: 2,
+          customFieldCount: 4,
+          incompatibleFieldCount: 3,
+          incompatibleFieldMappingItems: [
+            {
+              fieldName: 'host.name',
+              expectedValue: 'keyword',
+              actualValue: 'text',
+              description:
+                'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
+            },
+            {
+              fieldName: 'source.ip',
+              expectedValue: 'ip',
+              actualValue: 'text',
+              description: 'IP address of the source (IPv4 or IPv6).',
+            },
+          ],
+          incompatibleFieldValueItems: [
+            {
+              fieldName: 'event.category',
+              expectedValues: [
+                'authentication',
+                'configuration',
+                'database',
+                'driver',
+                'email',
+                'file',
+                'host',
+                'iam',
+                'intrusion_detection',
+                'malware',
+                'network',
+                'package',
+                'process',
+                'registry',
+                'session',
+                'threat',
+                'vulnerability',
+                'web',
+              ],
+              actualValues: [
+                { name: 'an_invalid_category', count: 2 },
+                { name: 'theory', count: 1 },
+              ],
+              description:
+                'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
+            },
+          ],
+          sameFamilyFieldCount: 0,
+          sameFamilyFields: [],
+          sameFamilyFieldItems: [],
+          unallowedMappingFields: ['host.name', 'source.ip'],
+          unallowedValueFields: ['event.category'],
+          sizeInBytes: 500000000,
+          ilmPhase: 'hot',
+          markdownComments: [
+            '### .ds-packetbeat-1\n',
+            '| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .ds-packetbeat-1 | 1,000,000 (100.0%) | 3 | `hot` | 476.8MB |\n\n',
+            '### **Incompatible fields** `3` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
+            "#### 3 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.11.0.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n",
+            '\n#### Incompatible field mappings - .ds-packetbeat-1\n\n\n| Field | ECS mapping type (expected) | Index mapping type (actual) | \n|-------|-----------------------------|-----------------------------|\n| host.name | `keyword` | `text` |\n| source.ip | `ip` | `text` |\n\n#### Incompatible field values - .ds-packetbeat-1\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `an_invalid_category` (2), `theory` (1) |\n\n',
+          ],
+          ecsVersion: '8.11.0',
+          indexId: 'uuid-1',
+          error: null,
+        });
+      });
+
+      describe('when isILMAvailable is false', () => {
+        it('should omit ilmPhase and nullify sizeInBytes when storing payload', () => {
+          const { result } = renderHook(() =>
+            useResultsRollup({
+              httpFetch,
+              toasts,
+              patterns,
+              isILMAvailable: false,
+              telemetryEvents: mockTelemetryEvents,
+            })
+          );
+
+          const patternRollup = getPatternRollupStub('packetbeat-*', 1, false);
+
+          act(() => {
+            result.current.updatePatternRollup(patternRollup);
+          });
+
+          jest.advanceTimersByTime(1000);
+
+          const mockOnCheckCompletedOpts = {
+            batchId: 'test-batch',
+            checkAllStartTime: Date.now(),
+            error: null,
+            formatBytes,
+            formatNumber,
+            indexName: '.ds-packetbeat-1',
+            partitionedFieldMetadata: mockPartitionedFieldMetadata,
+            pattern: 'packetbeat-*',
+            requestTime: 1500,
+            isLastCheck: true,
+            isCheckAll: true,
+          };
+
+          jest.advanceTimersByTime(1000);
+
+          act(() => {
+            result.current.onCheckCompleted(mockOnCheckCompletedOpts);
+          });
+
+          expect(mockTelemetryEvents.reportDataQualityIndexChecked).toHaveBeenCalledWith({
+            batchId: 'test-batch',
+            ecsVersion: '8.11.0',
+            errorCount: 0,
+            ilmPhase: undefined,
+            indexId: 'uuid-1',
+            indexName: '.ds-packetbeat-1',
+            isCheckAll: true,
+            numberOfCustomFields: 4,
+            numberOfDocuments: 1000000,
+            numberOfEcsFields: 2,
+            numberOfFields: 9,
+            numberOfIncompatibleFields: 3,
+            numberOfIndices: 1,
+            numberOfIndicesChecked: 1,
+            numberOfSameFamily: 0,
+            sameFamilyFields: [],
+            sizeInBytes: undefined,
+            timeConsumedMs: 1500,
+            unallowedMappingFields: ['host.name', 'source.ip'],
+            unallowedValueFields: ['event.category'],
+          });
+          expect(mockTelemetryEvents.reportDataQualityCheckAllCompleted).toHaveBeenCalledWith({
+            batchId: 'test-batch',
+            ecsVersion: '8.11.0',
+            isCheckAll: true,
+            numberOfDocuments: 1000000,
+            numberOfIncompatibleFields: 3,
+            numberOfIndices: 1,
+            numberOfIndicesChecked: 1,
+            numberOfSameFamily: 0,
+            sizeInBytes: undefined,
+            timeConsumedMs: 1000,
+          });
+
+          expect(httpFetch).toHaveBeenCalledWith('/internal/ecs_data_quality_dashboard/results', {
+            method: 'POST',
+            version: '1',
+            signal: expect.any(AbortSignal),
+            body: expect.any(String),
+          });
+
+          const body = JSON.parse(httpFetch.mock.calls[0][1].body);
+
+          expect(body).toEqual({
+            batchId: 'test-batch',
+            indexName: '.ds-packetbeat-1',
+            indexPattern: 'packetbeat-*',
+            isCheckAll: true,
+            checkedAt: new Date('2021-10-07T00:00:02Z').getTime(),
+            docsCount: 1000000,
+            totalFieldCount: 9,
+            ecsFieldCount: 2,
+            customFieldCount: 4,
+            incompatibleFieldCount: 3,
+            incompatibleFieldMappingItems: [
+              {
+                fieldName: 'host.name',
+                expectedValue: 'keyword',
+                actualValue: 'text',
+                description:
+                  'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
+              },
+              {
+                fieldName: 'source.ip',
+                expectedValue: 'ip',
+                actualValue: 'text',
+                description: 'IP address of the source (IPv4 or IPv6).',
+              },
+            ],
+            incompatibleFieldValueItems: [
+              {
+                fieldName: 'event.category',
+                expectedValues: [
+                  'authentication',
+                  'configuration',
+                  'database',
+                  'driver',
+                  'email',
+                  'file',
+                  'host',
+                  'iam',
+                  'intrusion_detection',
+                  'malware',
+                  'network',
+                  'package',
+                  'process',
+                  'registry',
+                  'session',
+                  'threat',
+                  'vulnerability',
+                  'web',
+                ],
+                actualValues: [
+                  { name: 'an_invalid_category', count: 2 },
+                  { name: 'theory', count: 1 },
+                ],
+                description:
+                  'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
+              },
+            ],
+            sameFamilyFieldCount: 0,
+            sameFamilyFields: [],
+            sameFamilyFieldItems: [],
+            unallowedMappingFields: ['host.name', 'source.ip'],
+            unallowedValueFields: ['event.category'],
+            ilmPhase: undefined,
+            sizeInBytes: 0,
+            markdownComments: [
+              '### .ds-packetbeat-1\n',
+              '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | .ds-packetbeat-1 | 1,000,000 (100.0%) | 3 |\n\n',
+              '### **Incompatible fields** `3` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
+              "#### 3 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.11.0.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n",
+              '\n#### Incompatible field mappings - .ds-packetbeat-1\n\n\n| Field | ECS mapping type (expected) | Index mapping type (actual) | \n|-------|-----------------------------|-----------------------------|\n| host.name | `keyword` | `text` |\n| source.ip | `ip` | `text` |\n\n#### Incompatible field values - .ds-packetbeat-1\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `an_invalid_category` (2), `theory` (1) |\n\n',
+            ],
+            ecsVersion: '8.11.0',
+            indexId: 'uuid-1',
+            error: null,
+          });
+        });
+      });
+    });
+
+    describe('when check fails with error message and no partitionedFieldMetadata', () => {
+      it('should update patternRollup with error message, reset state without persisting in storage', () => {
+        const { result } = renderHook(() =>
+          useResultsRollup({
+            httpFetch,
+            toasts,
+            patterns,
+            isILMAvailable,
+            telemetryEvents: mockTelemetryEvents,
+          })
+        );
+
+        const patternRollup = getPatternRollupStub('packetbeat-*', 1);
+
+        act(() => {
+          result.current.updatePatternRollup(patternRollup);
+        });
+
+        const mockOnCheckCompletedOpts = {
+          batchId: 'test-batch',
+          checkAllStartTime: Date.now(),
+          error: 'Something went wrong',
+          formatBytes,
+          formatNumber,
+          indexName: '.ds-packetbeat-1',
+          partitionedFieldMetadata: null,
+          pattern: 'packetbeat-*',
+          requestTime: 1500,
+          isLastCheck: true,
+          isCheckAll: true,
+        };
+
+        act(() => {
+          result.current.onCheckCompleted(mockOnCheckCompletedOpts);
+        });
+
+        expect(result.current.patternRollups['packetbeat-*'].results?.['.ds-packetbeat-1']).toEqual(
+          {
+            checkedAt: undefined,
+            docsCount: 1000000,
+            error: 'Something went wrong',
+            ilmPhase: 'hot',
+            incompatible: undefined,
+            indexName: '.ds-packetbeat-1',
+            markdownComments: expect.any(Array),
+            pattern: 'packetbeat-*',
+            sameFamily: undefined,
+          }
+        );
+
+        expect(mockTelemetryEvents.reportDataQualityIndexChecked).not.toHaveBeenCalled();
+
+        expect(httpFetch).not.toHaveBeenCalledWith(
+          '/internal/ecs_data_quality_dashboard/results',
+          expect.any(Object)
+        );
+      });
+    });
+
+    describe('edge cases', () => {
+      describe('given no error nor partitionedFieldMetadata', () => {
+        it('should reset result state accordingly and not invoke telemetry report nor persist in storage', () => {
+          const { result } = renderHook(() =>
+            useResultsRollup({
+              httpFetch,
+              toasts,
+              patterns,
+              isILMAvailable,
+              telemetryEvents: mockTelemetryEvents,
+            })
+          );
+
+          const patternRollup = getPatternRollupStub('packetbeat-*', 1);
+
+          act(() => {
+            result.current.updatePatternRollup(patternRollup);
+          });
+
+          const mockOnCheckCompletedOpts = {
+            batchId: 'test-batch',
+            checkAllStartTime: Date.now(),
+            error: null,
+            formatBytes,
+            formatNumber,
+            indexName: '.ds-packetbeat-1',
+            partitionedFieldMetadata: null,
+            pattern: 'packetbeat-*',
+            requestTime: 1500,
+            isLastCheck: true,
+            isCheckAll: true,
+          };
+
+          act(() => {
+            result.current.onCheckCompleted(mockOnCheckCompletedOpts);
+          });
+
+          expect(
+            result.current.patternRollups['packetbeat-*'].results?.['.ds-packetbeat-1']
+          ).toEqual({
+            checkedAt: undefined,
+            docsCount: 1000000,
+            error: null,
+            ilmPhase: 'hot',
+            incompatible: undefined,
+            indexName: '.ds-packetbeat-1',
+            markdownComments: expect.any(Array),
+            pattern: 'packetbeat-*',
+            sameFamily: undefined,
+          });
+
+          expect(mockTelemetryEvents.reportDataQualityIndexChecked).not.toHaveBeenCalled();
+
+          expect(httpFetch).not.toHaveBeenCalledWith(
+            '/internal/ecs_data_quality_dashboard/results',
+            expect.any(Object)
+          );
+        });
+      });
+    });
+  });
+
+  describe('calculating totals', () => {
+    describe('when patternRollups change', () => {
+      it('should update totals', () => {
+        const { result } = renderHook(() =>
+          useResultsRollup({
+            httpFetch,
+            toasts,
+            patterns: ['packetbeat-*', 'auditbeat-*'],
+            isILMAvailable,
+            telemetryEvents: mockTelemetryEvents,
+          })
+        );
+
+        const patternRollup1 = getPatternRollupStub('packetbeat-*', 1);
+        const patternRollup2 = getPatternRollupStub('auditbeat-*', 1);
+
+        expect(result.current.totalIndices).toBe(0);
+        expect(result.current.totalDocsCount).toBe(0);
+        expect(result.current.totalSizeInBytes).toBe(0);
+
+        act(() => {
+          result.current.updatePatternRollup(patternRollup1);
+        });
+
+        expect(result.current.totalIndices).toEqual(1);
+        expect(result.current.totalDocsCount).toEqual(1000000);
+        expect(result.current.totalSizeInBytes).toEqual(500000000);
+
+        act(() => {
+          result.current.updatePatternRollup(patternRollup2);
+        });
+
+        expect(result.current.totalIndices).toEqual(2);
+        expect(result.current.totalDocsCount).toEqual(2000000);
+        expect(result.current.totalSizeInBytes).toEqual(1000000000);
+      });
+    });
+  });
+});
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx
index 28b36765a245b..d95f1d1b7f20f 100644
--- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx
@@ -21,83 +21,29 @@ import {
   getTotalPatternSameFamily,
   getIndexId,
 } from './utils/stats';
-import {
-  getStorageResults,
-  postStorageResult,
-  formatStorageResult,
-  formatResultFromStorage,
-} from './utils/storage';
+import { postStorageResult, formatStorageResult } from './utils/storage';
 import { getPatternRollupsWithLatestCheckResult } from './utils/get_pattern_rollups_with_latest_check_result';
-import type {
-  DataQualityCheckResult,
-  OnCheckCompleted,
-  PatternRollup,
-  TelemetryEvents,
-} from '../../types';
+import type { OnCheckCompleted, PatternRollup, TelemetryEvents } from '../../types';
 import {
   getEscapedIncompatibleMappingsFields,
   getEscapedIncompatibleValuesFields,
   getEscapedSameFamilyFields,
 } from './utils/metadata';
 import { UseResultsRollupReturnValue } from './types';
-import { useIsMountedRef } from '../use_is_mounted_ref';
 import { getDocsCount, getIndexIncompatible, getSizeInBytes } from '../../utils/stats';
 import { getIlmPhase } from '../../utils/get_ilm_phase';
+import { useStoredPatternResults } from './hooks/use_stored_pattern_results';
 
 interface Props {
-  ilmPhases: string[];
   patterns: string[];
   toasts: IToasts;
   httpFetch: HttpHandler;
   telemetryEvents: TelemetryEvents;
   isILMAvailable: boolean;
 }
-const useStoredPatternResults = (patterns: string[], toasts: IToasts, httpFetch: HttpHandler) => {
-  const { isMountedRef } = useIsMountedRef();
-  const [storedPatternResults, setStoredPatternResults] = useState<
-    Array<{ pattern: string; results: Record<string, DataQualityCheckResult> }>
-  >([]);
-
-  useEffect(() => {
-    if (isEmpty(patterns)) {
-      return;
-    }
-
-    let ignore = false;
-    const abortController = new AbortController();
-    const fetchStoredPatternResults = async () => {
-      const requests = patterns.map((pattern) =>
-        getStorageResults({ pattern, httpFetch, abortController, toasts }).then((results = []) => ({
-          pattern,
-          results: Object.fromEntries(
-            results.map((storageResult) => [
-              storageResult.indexName,
-              formatResultFromStorage({ storageResult, pattern }),
-            ])
-          ),
-        }))
-      );
-      const patternResults = await Promise.all(requests);
-      if (patternResults?.length && !ignore) {
-        if (isMountedRef.current) {
-          setStoredPatternResults(patternResults);
-        }
-      }
-    };
-
-    fetchStoredPatternResults();
-    return () => {
-      ignore = true;
-    };
-  }, [httpFetch, isMountedRef, patterns, toasts]);
-
-  return storedPatternResults;
-};
-
 export const useResultsRollup = ({
   httpFetch,
   toasts,
-  ilmPhases,
   patterns,
   isILMAvailable,
   telemetryEvents,
@@ -247,12 +193,6 @@ export const useResultsRollup = ({
     [httpFetch, isILMAvailable, telemetryEvents, toasts]
   );
 
-  useEffect(() => {
-    // reset all state
-    setPatternRollups({});
-    setPatternIndexNames({});
-  }, [ilmPhases, patterns]);
-
   const useResultsRollupReturnValue = useMemo(
     () => ({
       onCheckCompleted,
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx
index 7d1a106d83570..b6d2736d7e175 100644
--- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx
@@ -104,7 +104,6 @@ const DataQualityPanelComponent: React.FC<Props> = ({
   );
 
   const resultsRollupHookReturnValue = useResultsRollup({
-    ilmPhases,
     patterns,
     httpFetch,
     toasts,
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/format.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/format.ts
new file mode 100644
index 0000000000000..844b573b61cad
--- /dev/null
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/format.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import numeral from '@elastic/numeral';
+
+import { EMPTY_STAT } from '../../../constants';
+
+const defaultBytesFormat = '0,0.[0]b';
+export const formatBytes = (value: number | undefined) =>
+  value != null ? numeral(value).format(defaultBytesFormat) : EMPTY_STAT;
+
+const defaultNumberFormat = '0,0.[000]';
+export const formatNumber = (value: number | undefined) =>
+  value != null ? numeral(value).format(defaultNumberFormat) : EMPTY_STAT;
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts
index 264198e510b5e..a8df6818605a1 100644
--- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts
@@ -5,10 +5,9 @@
  * 2.0.
  */
 
-import numeral from '@elastic/numeral';
-
 import { DataQualityProviderProps } from '../../../data_quality_context';
-import { EMPTY_STAT } from '../../../constants';
+
+import { formatBytes as formatBytesMock, formatNumber as formatNumberMock } from './format';
 
 export const getMergedDataQualityContextProps = (
   dataQualityContextProps?: Partial<DataQualityProviderProps>
@@ -36,10 +35,8 @@ export const getMergedDataQualityContextProps = (
     addSuccessToast: jest.fn(),
     canUserCreateAndReadCases: jest.fn(() => true),
     endDate: null,
-    formatBytes: (value: number | undefined) =>
-      value != null ? numeral(value).format('0,0.[0]b') : EMPTY_STAT,
-    formatNumber: (value: number | undefined) =>
-      value != null ? numeral(value).format('0,0.[000]') : EMPTY_STAT,
+    formatBytes: formatBytesMock,
+    formatNumber: formatNumberMock,
     isAssistantEnabled: true,
     lastChecked: '2023-03-28T22:27:28.159Z',
     openCreateCaseFlyout: jest.fn(),
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stub/get_pattern_rollup_stub/index.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stub/get_pattern_rollup_stub/index.ts
new file mode 100644
index 0000000000000..38aa129a6ec9a
--- /dev/null
+++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stub/get_pattern_rollup_stub/index.ts
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PatternRollup } from '../../types';
+
+const phases = ['hot', 'warm', 'cold', 'frozen'] as const;
+
+/**
+ *
+ * This function derives ilmExplain, results, stats and ilmExplainPhaseCounts
+ * from the provided pattern and indicesCount for the purpose of simplifying
+ * stubbing of resultsRollup in tests.
+ *
+ * @param pattern - The index pattern to simulate. Defaults to `'packetbeat-*'`.
+ * @param indicesCount - The number of indices to generate. Defaults to `2`.
+ * @param isILMAvailable - Whether ILM is available. Defaults to `true`.
+ * @returns An object containing stubbed pattern rollup data
+ */
+export const getPatternRollupStub = (
+  pattern = 'packetbeat-*',
+  indicesCount = 2,
+  isILMAvailable = true
+): PatternRollup => {
+  // Derive ilmExplain from isILMAvailable, pattern and indicesCount
+  const ilmExplain = isILMAvailable
+    ? Object.fromEntries(
+        Array.from({ length: indicesCount }).map((_, i) => {
+          const indexName = pattern.replace('*', `${i + 1}`);
+          const dsIndexName = `.ds-${indexName}`;
+          // Cycle through phases
+          const phase = phases[i % phases.length];
+          return [
+            dsIndexName,
+            {
+              index: dsIndexName,
+              managed: true,
+              policy: pattern,
+              phase,
+            },
+          ];
+        })
+      )
+    : null;
+
+  // Derive ilmExplainPhaseCounts from ilmExplain
+  const ilmExplainPhaseCounts = ilmExplain
+    ? phases.reduce(
+        (counts, phase) => ({
+          ...counts,
+          [phase]: Object.values(ilmExplain).filter((explain) => explain.phase === phase).length,
+        }),
+        { hot: 0, warm: 0, cold: 0, frozen: 0, unmanaged: 0 }
+      )
+    : undefined;
+
+  // Derive results from pattern and indicesCount
+  const results = Object.fromEntries(
+    Array.from({ length: indicesCount }, (_, i) => {
+      const indexName = pattern.replace('*', `${i + 1}`);
+      const dsIndexName = `.ds-${indexName}`;
+      return [
+        dsIndexName,
+        {
+          docsCount: 1000000 + i * 100000, // Example doc count
+          error: null,
+          ilmPhase: ilmExplain?.[dsIndexName].phase,
+          incompatible: i,
+          indexName: dsIndexName,
+          markdownComments: ['foo', 'bar', 'baz'],
+          pattern,
+          sameFamily: i,
+          checkedAt: Date.now(),
+        },
+      ];
+    })
+  );
+
+  // Derive stats from isILMAvailable, pattern and indicesCount
+  const stats = Object.fromEntries(
+    Array.from({ length: indicesCount }, (_, i) => {
+      const indexName = pattern.replace('*', `${i + 1}`);
+      const dsIndexName = `.ds-${indexName}`;
+      return [
+        dsIndexName,
+        {
+          uuid: `uuid-${i + 1}`,
+          size_in_bytes: isILMAvailable ? 500000000 + i * 10000000 : null,
+          name: dsIndexName,
+          num_docs: results[dsIndexName].docsCount,
+        },
+      ];
+    })
+  );
+
+  // Derive total docsCount and sizeInBytes from stats
+  const totalDocsCount = Object.values(stats).reduce((sum, stat) => sum + stat.num_docs, 0);
+  const totalSizeInBytes = isILMAvailable
+    ? Object.values(stats).reduce((sum, stat) => sum + (stat.size_in_bytes ?? 0), 0)
+    : undefined;
+
+  return {
+    docsCount: totalDocsCount,
+    error: null,
+    pattern,
+    ilmExplain,
+    ilmExplainPhaseCounts,
+    indices: indicesCount,
+    results,
+    sizeInBytes: totalSizeInBytes,
+    stats,
+  };
+};