[Index management] Project level retention support (#193715)

(cherry picked from commit ac2a5d2)

config/serverless.security.yml

@@ -119,6 +119,9 @@ xpack.ml.compatibleModuleType: 'security'
 # Disable the embedded Dev Console
 console.ui.embeddedEnabled: false
 
+# Enable project level rentention checks in DSL form from Index Management UI
+xpack.index_management.enableProjectLevelRetentionChecks: true
+
 # Experimental Security Solution features
 
 # This feature is disabled in Serverless until fully performance tested within a Serverless environment

config/serverless.yml

@@ -110,6 +110,8 @@ xpack.index_management.editableIndexSettings: limited
 xpack.index_management.enableMappingsSourceFieldSection: false
 # Disable toggle for enabling data retention in DSL form from Index Management UI
 xpack.index_management.enableTogglingDataRetention: false
+# Disable project level rentention checks in DSL form from Index Management UI
+xpack.index_management.enableProjectLevelRetentionChecks: false
 
 # Disable Manage Processors UI in Ingest Pipelines
 xpack.ingest_pipelines.enableManageProcessors: false

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -273,6 +273,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.index_management.ui.enabled (boolean?)',
         'xpack.infra.sources.default.fields.message (array?)',
         'xpack.index_management.enableTogglingDataRetention (boolean?|never)',
+        'xpack.index_management.enableProjectLevelRetentionChecks (boolean?|never)',
         'xpack.integration_assistant.enableExperimental (array?)',
         /**
          * Feature flags bellow are conditional based on traditional/serverless offering

x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts

@@ -100,6 +100,7 @@ export type TestSubjects =
   | 'configuredByILMWarning'
   | 'show-filters-button'
   | 'filter-option-h'
+  | 'filter-option-d'
   | 'infiniteRetentionPeriod.input'
   | 'saveButton'
   | 'dsIsFullyManagedByILM'

x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_project_level_retention.test.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { act } from 'react-dom/test-utils';
+import { createMemoryHistory } from 'history';
+import { notificationServiceMock } from '@kbn/core/public/mocks';
+
+import { breadcrumbService } from '../../../public/application/services/breadcrumbs';
+import { MAX_DATA_RETENTION } from '../../../common/constants';
+import * as fixtures from '../../../test/fixtures';
+import { setupEnvironment } from '../helpers';
+import { notificationService } from '../../../public/application/services/notification';
+
+import {
+  DataStreamsTabTestBed,
+  setup,
+  createDataStreamPayload,
+  createDataStreamBackingIndex,
+  createNonDataStreamIndex,
+} from './data_streams_tab.helpers';
+
+const urlServiceMock = {
+  locators: {
+    get: () => ({
+      getLocation: async () => ({
+        app: '',
+        path: '',
+        state: {},
+      }),
+      getUrl: async ({ policyName }: { policyName: string }) => `/test/${policyName}`,
+      navigate: async () => {},
+      useUrl: () => '',
+    }),
+  },
+};
+
+describe('Data Streams - Project level max retention', () => {
+  const { httpSetup, httpRequestsMockHelpers } = setupEnvironment();
+  let testBed: DataStreamsTabTestBed;
+  jest.spyOn(breadcrumbService, 'setBreadcrumbs');
+
+  const notificationsServiceMock = notificationServiceMock.createStartContract();
+
+  beforeEach(async () => {
+    const {
+      setLoadIndicesResponse,
+      setLoadDataStreamsResponse,
+      setLoadDataStreamResponse,
+      setLoadTemplateResponse,
+      setLoadTemplatesResponse,
+    } = httpRequestsMockHelpers;
+
+    setLoadIndicesResponse([
+      createDataStreamBackingIndex('data-stream-index', 'dataStream1'),
+      createNonDataStreamIndex('non-data-stream-index'),
+    ]);
+
+    const dataStreamForDetailPanel = createDataStreamPayload({
+      name: 'dataStream1',
+      storageSize: '5b',
+      storageSizeBytes: 5,
+      // metering API mock
+      meteringStorageSize: '156kb',
+      meteringStorageSizeBytes: 156000,
+      meteringDocsCount: 10000,
+    });
+
+    setLoadDataStreamsResponse([
+      dataStreamForDetailPanel,
+      createDataStreamPayload({
+        name: 'dataStream2',
+        storageSize: '1kb',
+        storageSizeBytes: 1000,
+        // metering API mock
+        meteringStorageSize: '156kb',
+        meteringStorageSizeBytes: 156000,
+        meteringDocsCount: 10000,
+        lifecycle: {
+          enabled: true,
+          data_retention: '7d',
+          effective_retention: '5d',
+          globalMaxRetention: '20d',
+          retention_determined_by: MAX_DATA_RETENTION,
+        },
+      }),
+    ]);
+
+    setLoadDataStreamResponse(dataStreamForDetailPanel.name, dataStreamForDetailPanel);
+
+    const indexTemplate = fixtures.getTemplate({ name: 'indexTemplate' });
+    setLoadTemplatesResponse({ templates: [indexTemplate], legacyTemplates: [] });
+    setLoadTemplateResponse(indexTemplate.name, indexTemplate);
+
+    notificationService.setup(notificationsServiceMock);
+    testBed = await setup(httpSetup, {
+      history: createMemoryHistory(),
+      services: {
+        notificationService,
+      },
+      config: {
+        enableProjectLevelRetentionChecks: true,
+      },
+    });
+    await act(async () => {
+      testBed.actions.goToDataStreamsList();
+    });
+    testBed.component.update();
+  });
+
+  it('Should show error when retention value is bigger than project level retention', async () => {
+    const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;
+
+    const ds1 = createDataStreamPayload({
+      name: 'dataStream1',
+      lifecycle: {
+        enabled: true,
+        data_retention: '25d',
+        effective_retention: '25d',
+        retention_determined_by: MAX_DATA_RETENTION,
+        globalMaxRetention: '20d',
+      },
+    });
+
+    setLoadDataStreamsResponse([ds1]);
+    setLoadDataStreamResponse(ds1.name, ds1);
+
+    testBed = await setup(httpSetup, {
+      history: createMemoryHistory(),
+      url: urlServiceMock,
+      config: {
+        enableProjectLevelRetentionChecks: true,
+      },
+    });
+    await act(async () => {
+      testBed.actions.goToDataStreamsList();
+    });
+    testBed.component.update();
+
+    const { actions } = testBed;
+
+    await actions.clickNameAt(0);
+    actions.clickEditDataRetentionButton();
+
+    expect(testBed.form.getErrorsMessages().length).toBeGreaterThan(0);
+  });
+});

x-pack/plugins/index_management/common/lib/data_stream_serialization.ts

@@ -23,6 +23,7 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
     privileges,
     hidden,
     lifecycle,
+    global_max_retention: globalMaxRetention,
     next_generation_managed_by: nextGenerationManagedBy,
   } = dataStreamFromEs;
 
@@ -57,7 +58,10 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
     _meta,
     privileges,
     hidden,
-    lifecycle,
+    lifecycle: {
+      ...lifecycle,
+      globalMaxRetention,
+    },
     nextGenerationManagedBy,
   };
 }

x-pack/plugins/index_management/common/types/data_streams.ts

@@ -34,6 +34,7 @@ export type DataStreamIndexFromEs = IndicesDataStreamIndex;
 export type Health = 'green' | 'yellow' | 'red';
 
 export interface EnhancedDataStreamFromEs extends IndicesDataStream {
+  global_max_retention?: string;
   store_size?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size'];
   store_size_bytes?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size_bytes'];
   maximum_timestamp?: IndicesDataStreamsStatsDataStreamsStatsItem['maximum_timestamp'];
@@ -63,6 +64,7 @@ export interface DataStream {
     enabled?: boolean;
     effective_retention?: string;
     retention_determined_by?: string;
+    globalMaxRetention?: string;
   };
 }
 

x-pack/plugins/index_management/public/application/app_context.tsx

@@ -68,6 +68,7 @@ export interface AppDependencies {
     editableIndexSettings: 'all' | 'limited';
     enableMappingsSourceFieldSection: boolean;
     enableTogglingDataRetention: boolean;
+    enableProjectLevelRetentionChecks: boolean;
     enableSemanticText: boolean;
   };
   history: ScopedHistory;

x-pack/plugins/index_management/public/application/hooks/use_state_with_localstorage.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Dispatch, SetStateAction, useEffect, useState } from 'react';
+
+function parseJsonOrDefault<Obj>(value: string | null, defaultValue: Obj): Obj {
+  if (!value) {
+    return defaultValue;
+  }
+  try {
+    return JSON.parse(value) as Obj;
+  } catch (e) {
+    return defaultValue;
+  }
+}
+
+export function useStateWithLocalStorage<State>(
+  key: string,
+  defaultState: State
+): [State, Dispatch<SetStateAction<State>>] {
+  const storageState = localStorage.getItem(key);
+  const [state, setState] = useState<State>(parseJsonOrDefault<State>(storageState, defaultState));
+  useEffect(() => {
+    localStorage.setItem(key, JSON.stringify(state));
+  }, [key, state]);
+  return [state, setState];
+}