Dynamic ecs

x-pack/plugins/alerting/server/alerts_service/alerts_service.ts

@@ -8,7 +8,7 @@
 import { isEmpty, isEqual, omit } from 'lodash';
 import { Logger, ElasticsearchClient } from '@kbn/core/server';
 import { filter, firstValueFrom, Observable } from 'rxjs';
-import { alertFieldMap, ecsFieldMap, legacyAlertFieldMap } from '@kbn/alerts-as-data-utils';
+import { alertFieldMap, legacyAlertFieldMap } from '@kbn/alerts-as-data-utils';
 import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server';
 import {
   DEFAULT_ALERTS_ILM_POLICY_NAME,
@@ -48,9 +48,9 @@ import { IAlertsClient } from '../alerts_client/types';
 import { setAlertsToUntracked, SetAlertsToUntrackedParams } from './lib/set_alerts_to_untracked';
 
 export const TOTAL_FIELDS_LIMIT = 2500;
+export const ECS_COMPONENT_TEMPLATE_NAME = 'ecs@mappings';
 const LEGACY_ALERT_CONTEXT = 'legacy-alert';
-export const ECS_CONTEXT = `ecs`;
-export const ECS_COMPONENT_TEMPLATE_NAME = getComponentTemplateName({ name: ECS_CONTEXT });
+
 interface AlertsServiceParams {
   logger: Logger;
   pluginStop$: Observable<void>;
@@ -348,17 +348,6 @@ export class AlertsService implements IAlertsService {
             }),
             totalFieldsLimit: TOTAL_FIELDS_LIMIT,
           }),
-        () =>
-          createOrUpdateComponentTemplate({
-            logger: this.options.logger,
-            esClient,
-            template: getComponentTemplate({
-              fieldMap: ecsFieldMap,
-              name: ECS_CONTEXT,
-              includeSettings: true,
-            }),
-            totalFieldsLimit: TOTAL_FIELDS_LIMIT,
-          }),
       ];
 
       // Install in parallel
@@ -414,9 +403,9 @@ export class AlertsService implements IAlertsService {
     // 4. Framework common component template, always included
     const componentTemplateRefs: string[] = [];
 
-    // If useEcs is set to true, add the ECS component template to the references
+    // If useEcs is set to true, add the ECS dynamic mapping template to the references
     if (useEcs) {
-      componentTemplateRefs.push(getComponentTemplateName({ name: ECS_CONTEXT }));
+      componentTemplateRefs.push(ECS_COMPONENT_TEMPLATE_NAME);
     }
 
     // If fieldMap is not empty, create a context specific component template and add to the references

x-pack/plugins/alerting/server/alerts_service/index.ts

@@ -9,7 +9,7 @@ export {
   DEFAULT_ALERTS_ILM_POLICY,
   DEFAULT_ALERTS_ILM_POLICY_NAME,
 } from './default_lifecycle_policy';
-export { ECS_COMPONENT_TEMPLATE_NAME, ECS_CONTEXT, TOTAL_FIELDS_LIMIT } from './alerts_service';
+export { ECS_COMPONENT_TEMPLATE_NAME, TOTAL_FIELDS_LIMIT } from './alerts_service';
 export { getComponentTemplate, VALID_ALERT_INDEX_PREFIXES } from './resource_installer_utils';
 export {
   type InitializationPromise,

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.test.ts

@@ -35,7 +35,7 @@ const IndexTemplate = (namespace: string = 'default', useDataStream: boolean = f
           managed: true,
           namespace,
         },
-        dynamic: false,
+        dynamic: true,
       },
       settings: {
         auto_expand_replicas: '0-1',
@@ -50,6 +50,7 @@ const IndexTemplate = (namespace: string = 'default', useDataStream: boolean = f
             }),
         'index.mapping.ignore_malformed': true,
         'index.mapping.total_fields.limit': 2500,
+        'index.mapping.total_fields.ignore_dynamic_beyond_limit': true,
       },
     },
     priority: namespace.length,

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.ts

@@ -70,9 +70,10 @@ export const getIndexTemplate = ({
               }),
           'index.mapping.ignore_malformed': true,
           'index.mapping.total_fields.limit': totalFieldsLimit,
+          'index.mapping.total_fields.ignore_dynamic_beyond_limit': true,
         },
         mappings: {
-          dynamic: false,
+          dynamic: true,
           _meta: indexMetadata,
         },
         ...(indexPatterns.secondaryAlias

x-pack/plugins/alerting/server/index.ts

@@ -55,7 +55,6 @@ export {
   DEFAULT_ALERTS_ILM_POLICY,
   DEFAULT_ALERTS_ILM_POLICY_NAME,
   ECS_COMPONENT_TEMPLATE_NAME,
-  ECS_CONTEXT,
   TOTAL_FIELDS_LIMIT,
   VALID_ALERT_INDEX_PREFIXES,
   getComponentTemplate,

x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts

@@ -470,7 +470,7 @@ describe('Task Runner', () => {
         expect(call.services.alertsClient?.setAlertData).toBeTruthy();
         expect(call.services.scopedClusterClient).toBeTruthy();
         expect(call.services).toBeTruthy();
-        expect(logger.debug).toHaveBeenCalledTimes(useDataStreamForAlerts ? 9 : 10);
+        expect(logger.debug).toHaveBeenCalledTimes(useDataStreamForAlerts ? 8 : 9);
 
         let debugCall = 1;
         expect(logger.debug).nthCalledWith(
@@ -494,10 +494,6 @@ describe('Task Runner', () => {
           debugCall++,
           'Installing component template .alerts-legacy-alert-mappings'
         );
-        expect(logger.debug).nthCalledWith(
-          debugCall++,
-          'Installing component template .alerts-ecs-mappings'
-        );
         expect(logger.debug).nthCalledWith(
           debugCall++,
           'deprecated ruleRunStatus for test:1: {"lastExecutionDate":"1970-01-01T00:00:00.000Z","status":"ok"}',

x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts

@@ -17,7 +17,6 @@ import {
 
 import { Dataset } from './index_options';
 import { IndexInfo } from './index_info';
-import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server';
 import type { DataStreamAdapter } from '@kbn/alerting-plugin/server';
 import { getDataStreamAdapter } from '@kbn/alerting-plugin/server/alerts_service/lib/data_stream_adapter';
 
@@ -148,13 +147,9 @@ describe('resourceInstaller', () => {
           expect(mockClusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(
             useDataStreamForAlerts ? 0 : 1
           );
-          expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2);
+          expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1);
           expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenNthCalledWith(
             1,
-            expect.objectContaining({ name: ECS_COMPONENT_TEMPLATE_NAME })
-          );
-          expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenNthCalledWith(
-            2,
             expect.objectContaining({ name: TECHNICAL_COMPONENT_TEMPLATE_NAME })
           );
         });

x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts

@@ -15,7 +15,6 @@ import {
   createOrUpdateIndexTemplate,
   DEFAULT_ALERTS_ILM_POLICY,
   DEFAULT_ALERTS_ILM_POLICY_NAME,
-  ECS_COMPONENT_TEMPLATE_NAME,
   getIndexTemplate,
   installWithTimeout,
   TOTAL_FIELDS_LIMIT,
@@ -25,7 +24,6 @@ import {
 } from '@kbn/alerting-plugin/server';
 import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../common/assets';
 import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template';
-import { ecsComponentTemplate } from '../../common/assets/component_templates/ecs_component_template';
 import type { IndexInfo } from './index_info';
 
 interface ConstructorOptions {
@@ -89,15 +87,6 @@ export class ResourceInstaller {
                     policy: DEFAULT_ALERTS_ILM_POLICY,
                     dataStreamAdapter: this.options.dataStreamAdapter,
                   }),
-                  createOrUpdateComponentTemplate({
-                    logger,
-                    esClient: clusterClient,
-                    template: {
-                      name: ECS_COMPONENT_TEMPLATE_NAME,
-                      body: ecsComponentTemplate,
-                    },
-                    totalFieldsLimit: TOTAL_FIELDS_LIMIT,
-                  }),
                 ]),
             createOrUpdateComponentTemplate({
               logger,