[Fleet] added eventIngestedEnabled flag (elastic#199733)

## Summary

Closes elastic/integrations#11491

Added a separate flag `xpack.fleet.eventIngestedEnabled` (false by
default) to keep the `event.ingested` mapping even when
`agentIdVerificationEnabled` is disabled (in serverless oblt projects)

Created a new pipeline `.fleet_event_ingested_pipeline-1` to use when
only `eventIngestedEnabled` is enabled, to skip the step of calculating
`agent_id_status`.
I couldn't change `.fleet_final_pipeline-1` because the pipeline steps
have to be different based on the flags.

## To verify:
Note: After changing the flags, the packages have to be reinstalled to
see the changes in the index templates, tested with `elastic_agent`
package.
Also, the data streams should be rolled over to see the changes in the
ingested data.
```
POST logs-elastic_agent-default/_rollover
POST logs-elastic_agent.metricbeat-default/_rollover
```

### Default behaviour unchanged (Agent id verification enabled,
event.ingested flag disabled)
- by default: no change in behaviour, both `event.ingested` and
`event.agent_id_status` should be mapped

<img width="1381" alt="image"
src="https://github.com/user-attachments/assets/33c6fafc-1365-4e6a-b8fe-45f58a6c479e">
<img width="856" alt="image"
src="https://github.com/user-attachments/assets/54fefa62-bbb5-4ce5-a3dd-f56123e5e042">

### Agent id verification disabled, event.ingested enabled
- set in `kibana.yml`
```
xpack.fleet.agentIdVerificationEnabled: false
xpack.fleet.eventIngestedEnabled: true
```
- verify that `event.ingested` is mapped, `event.agent_id_status` is not

<img width="923" alt="image"
src="https://github.com/user-attachments/assets/0c18b3f2-6071-4f5a-a377-abeb4b4890ef">
<img width="1425" alt="image"
src="https://github.com/user-attachments/assets/2c93feca-a719-4cdb-983f-8f1269c22c88">
<img width="531" alt="image"
src="https://github.com/user-attachments/assets/b214143a-04a9-42d0-8ccc-07059d836039">
<img width="2342" alt="image"
src="https://github.com/user-attachments/assets/9247b6eb-e426-4eed-8d7c-3cb89be9dbdd">
<img width="2545" alt="image"
src="https://github.com/user-attachments/assets/d1b209cf-503d-47a0-ac06-1fd75395bab2">

### Agent id verification disabled, event.ingested disabled
- set in `kibana.yml`
```
xpack.fleet.agentIdVerificationEnabled: false
xpack.fleet.eventIngestedEnabled: false # default
```
- verify that neither `event.ingested` and `event.agent_id_status` is
mapped

<img width="522" alt="image"
src="https://github.com/user-attachments/assets/2434e4df-5b5e-45a5-a438-7b305834db63">

### Agent id verification enabled, event.ingested enabled
- set in `kibana.yml`
```
xpack.fleet.agentIdVerificationEnabled: true # default
xpack.fleet.eventIngestedEnabled: true
```
- both `event.ingested` and `event.agent_id_status` should be mapped

<img width="1284" alt="image"
src="https://github.com/user-attachments/assets/226838fe-8a2a-455f-812d-049d31fe4600">
<img width="858" alt="image"
src="https://github.com/user-attachments/assets/4fd99149-47f3-462c-b1ec-a2d45684560f">


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

.buildkite/ftr_platform_stateful_configs.yml

@@ -183,6 +183,7 @@ enabled:
   - x-pack/test/fleet_api_integration/config.agent.ts
   - x-pack/test/fleet_api_integration/config.agent_policy.ts
   - x-pack/test/fleet_api_integration/config.epm.ts
+  - x-pack/test/fleet_api_integration/config.event_ingested.ts
   - x-pack/test/fleet_api_integration/config.fleet.ts
   - x-pack/test/fleet_api_integration/config.package_policy.ts
   - x-pack/test/fleet_api_integration/config.space_awareness.ts

config/serverless.oblt.yml

@@ -129,6 +129,9 @@ xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability'
 ## Disable adding the component template `.fleet_agent_id_verification-1` to every index template for each datastream for each integration
 xpack.fleet.agentIdVerificationEnabled: false
 
+## Enable event.ingested separately because agentIdVerification is disabled
+xpack.fleet.eventIngestedEnabled: true
+
 ## Enable the capability for the observability feature ID in the serverless environment to take ownership of the rules.
 ## The value need to be a featureId observability Or stackAlerts Or siem
 xpack.alerting.rules.overwriteProducer: 'observability'

x-pack/plugins/fleet/common/types/index.ts

@@ -49,6 +49,7 @@ export interface FleetConfigType {
   packages?: PreconfiguredPackage[];
   outputs?: PreconfiguredOutput[];
   agentIdVerificationEnabled?: boolean;
+  eventIngestedEnabled?: boolean;
   enableExperimental?: string[];
   packageVerification?: {
     gpgKeyPath?: string;

x-pack/plugins/fleet/public/mock/plugin_configuration.ts

@@ -13,6 +13,7 @@ export const createConfigurationMock = (): FleetConfigType => {
     registryUrl: '',
     registryProxyUrl: '',
     agentIdVerificationEnabled: true,
+    eventIngestedEnabled: false,
     agents: {
       enabled: true,
       elasticsearch: {

x-pack/plugins/fleet/server/config.ts

@@ -170,6 +170,7 @@ export const config: PluginConfigDescriptor = {
       proxies: PreconfiguredFleetProxiesSchema,
       spaceSettings: PreconfiguredSpaceSettingsSchema,
       agentIdVerificationEnabled: schema.boolean({ defaultValue: true }),
+      eventIngestedEnabled: schema.boolean({ defaultValue: false }),
       setup: schema.maybe(
         schema.object({
           agentPolicySchemaUpgradeBatchSize: schema.maybe(schema.number()),

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -17,6 +17,8 @@ export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1';
 
 export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';
 
+export const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1';
+
 export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1';
 
 export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = {
@@ -46,6 +48,12 @@ export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = {
 };
 export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1';
 
+export const INGESTED_MAPPING = {
+  type: 'date',
+  format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+  ignore_malformed: false,
+};
+
 export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = {
   _meta: meta,
   template: {
@@ -58,11 +66,7 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = {
       properties: {
         event: {
           properties: {
-            ingested: {
-              type: 'date',
-              format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
-              ignore_malformed: false,
-            },
+            ingested: INGESTED_MAPPING,
             agent_id_status: {
               ignore_above: 1024,
               type: 'keyword',
@@ -74,12 +78,38 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = {
   },
 };
 
+export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME = '.fleet_event_ingested-1';
+
+export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT = {
+  _meta: meta,
+  template: {
+    settings: {
+      index: {
+        final_pipeline: FLEET_EVENT_INGESTED_PIPELINE_ID,
+      },
+    },
+    mappings: {
+      properties: {
+        event: {
+          properties: {
+            ingested: INGESTED_MAPPING,
+          },
+        },
+      },
+    },
+  },
+};
+
 export const FLEET_COMPONENT_TEMPLATES = [
   { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT },
   {
     name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
     body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT,
   },
+  {
+    name: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME,
+    body: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT,
+  },
 ];
 
 export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`;
@@ -96,6 +126,59 @@ export const STACK_COMPONENT_TEMPLATES = [
   STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
 ];
 
+export const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1;
+
+// If the content is updated you probably need to update the FLEET_EVENT_INGESTED_PIPELINE_VERSION too to allow upgrade of the pipeline
+export const FLEET_EVENT_INGESTED_PIPELINE_CONTENT = `---
+version: ${FLEET_EVENT_INGESTED_PIPELINE_VERSION}
+_meta:
+  managed_by: ${meta.managed_by}
+  managed: ${meta.managed}
+description: >
+  Pipeline for processing all incoming Fleet Agent documents that adds event.ingested.
+processors:
+  - script:
+      description: Add time when event was ingested (and remove sub-seconds to improve storage efficiency)
+      tag: truncate-subseconds-event-ingested
+      ignore_failure: true
+      source: |-
+        if (ctx?.event == null) {
+          ctx.event = [:];
+        }
+
+        ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+  - remove:
+      description: Remove any pre-existing untrusted values.
+      field:
+        - event.agent_id_status
+        - _security
+      ignore_missing: true
+  - remove:
+      description: Remove event.original unless the preserve_original_event tag is set
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
+  - set_security_user:
+      field: _security
+      properties:
+        - authentication_type
+        - username
+        - realm
+        - api_key
+  - remove:
+      field: _security
+      ignore_missing: true
+on_failure:
+  - remove:
+      field: _security
+      ignore_missing: true
+      ignore_failure: true
+  - append:
+      field: error.message
+      value:
+        - 'failed in Fleet agent event_ingested_pipeline: {{ _ingest.on_failure_message }}'`;
+
 export const FLEET_FINAL_PIPELINE_VERSION = 4;
 
 // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline

x-pack/plugins/fleet/server/constants/index.ts

@@ -111,6 +111,9 @@ export {
   FLEET_FINAL_PIPELINE_ID,
   FLEET_FINAL_PIPELINE_CONTENT,
   FLEET_FINAL_PIPELINE_VERSION,
+  FLEET_EVENT_INGESTED_PIPELINE_ID,
+  FLEET_EVENT_INGESTED_PIPELINE_VERSION,
+  FLEET_EVENT_INGESTED_PIPELINE_CONTENT,
   FLEET_INSTALL_FORMAT_VERSION,
   FLEET_AGENT_POLICIES_SCHEMA_VERSION,
   STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,

x-pack/plugins/fleet/server/mocks/index.ts

@@ -81,6 +81,7 @@ export const createAppContextStartContractMock = (
     agents: { enabled: true, elasticsearch: {} },
     enabled: true,
     agentIdVerificationEnabled: true,
+    eventIngestedEnabled: false,
     ...configOverrides,
   };
 
@@ -120,6 +121,7 @@ export const createAppContextStartContractMock = (
       agents: { enabled: true, elasticsearch: {} },
       enabled: true,
       agentIdVerificationEnabled: true,
+      eventIngestedEnabled: false,
     },
     config$,
     kibanaVersion: '8.99.0', // Fake version :)

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts

@@ -20,6 +20,9 @@ import {
   FLEET_FINAL_PIPELINE_CONTENT,
   FLEET_FINAL_PIPELINE_ID,
   FLEET_FINAL_PIPELINE_VERSION,
+  FLEET_EVENT_INGESTED_PIPELINE_ID,
+  FLEET_EVENT_INGESTED_PIPELINE_VERSION,
+  FLEET_EVENT_INGESTED_PIPELINE_CONTENT,
 } from '../../../../constants';
 import { getPipelineNameForDatastream } from '../../../../../common/services';
 import type { ArchiveEntry, PackageInstallContext } from '../../../../../common/types';
@@ -302,6 +305,39 @@ export async function ensureFleetFinalPipelineIsInstalled(
   return { isCreated: false };
 }
 
+export async function ensureFleetEventIngestedPipelineIsInstalled(
+  esClient: ElasticsearchClient,
+  logger: Logger
+) {
+  const esClientRequestOptions: TransportRequestOptions = {
+    ignore: [404],
+  };
+  const res = await esClient.ingest.getPipeline(
+    { id: FLEET_EVENT_INGESTED_PIPELINE_ID },
+    { ...esClientRequestOptions, meta: true }
+  );
+
+  const installedVersion = res?.body[FLEET_EVENT_INGESTED_PIPELINE_ID]?.version;
+  if (
+    res.statusCode === 404 ||
+    !installedVersion ||
+    installedVersion < FLEET_EVENT_INGESTED_PIPELINE_VERSION
+  ) {
+    await installPipeline({
+      esClient,
+      logger,
+      pipeline: {
+        nameForInstallation: FLEET_EVENT_INGESTED_PIPELINE_ID,
+        contentForInstallation: FLEET_EVENT_INGESTED_PIPELINE_CONTENT,
+        extension: 'yml',
+      },
+    });
+    return { isCreated: true };
+  }
+
+  return { isCreated: false };
+}
+
 const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/');
 
 const isDataStreamPipeline = (path: string, dataStreamDataset: string) => {

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -14,15 +14,18 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
 
 import { errors } from '@elastic/elasticsearch';
 
-import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets';
+import {
+  FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
+  FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME,
+  STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
+} from '../../../../constants/fleet_es_assets';
 
 import { createAppContextStartContractMock } from '../../../../mocks';
 import { appContextService } from '../../..';
 import type { RegistryDataStream } from '../../../../types';
 import { processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
 import {
-  FLEET_COMPONENT_TEMPLATES,
   STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
   FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
   STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
@@ -36,10 +39,6 @@ import {
   updateCurrentWriteIndices,
 } from './template';
 
-const FLEET_COMPONENT_TEMPLATES_NAMES = FLEET_COMPONENT_TEMPLATES.map(
-  (componentTemplate) => componentTemplate.name
-);
-
 // Add our own serialiser to just do JSON.stringify
 expect.addSnapshotSerializer({
   print(val) {
@@ -88,7 +87,8 @@ describe('EPM template', () => {
       STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
       ...composedOfTemplates,
       STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
-      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+      FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
+      FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
     ]);
   });
 
@@ -108,7 +108,8 @@ describe('EPM template', () => {
       'metrics@tsdb-settings',
       ...composedOfTemplates,
       STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
-      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+      FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
+      FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
     ]);
   });
 
@@ -138,6 +139,34 @@ describe('EPM template', () => {
     ]);
   });
 
+  it('creates fleet event ingested component template if event ingested flag is enabled', () => {
+    appContextService.start(
+      createAppContextStartContractMock({
+        agentIdVerificationEnabled: false,
+        eventIngestedEnabled: true,
+      })
+    );
+    const composedOfTemplates = ['component1', 'component2'];
+
+    const template = getTemplate({
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
+      packageName: 'nginx',
+      composedOfTemplates,
+      templatePriority: 200,
+      mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+    });
+    expect(template.composed_of).toStrictEqual([
+      STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
+      STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
+      ...composedOfTemplates,
+      STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
+      FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
+      FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME,
+    ]);
+  });
+
   it('adds empty composed_of correctly', () => {
     const composedOfTemplates: string[] = [];
 
@@ -154,7 +183,8 @@ describe('EPM template', () => {
       STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
       STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
       STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
-      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+      FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
+      FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME,
     ]);
   });
 

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -15,7 +15,10 @@ import type {
 import pMap from 'p-map';
 import { isResponseError } from '@kbn/es-errors';
 
-import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets';
+import {
+  FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME,
+  STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS,
+} from '../../../../constants/fleet_es_assets';
 
 import type { Field, Fields } from '../../fields/field';
 import type {
@@ -27,6 +30,7 @@ import type {
 } from '../../../../types';
 import { appContextService } from '../../..';
 import { getRegistryDataStreamAssetBaseName } from '../../../../../common/services';
+import type { FleetConfigType } from '../../../../../common/types';
 import {
   STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS,
   FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
@@ -115,6 +119,9 @@ export function getTemplate({
 
   const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries);
 
+  const isEventIngestedEnabled = (config?: FleetConfigType): boolean =>
+    Boolean(!config?.agentIdVerificationEnabled && config?.eventIngestedEnabled);
+
   template.composed_of = [
     ...esBaseComponents,
     ...(template.composed_of || []),
@@ -123,6 +130,9 @@ export function getTemplate({
     ...(appContextService.getConfig()?.agentIdVerificationEnabled
       ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME]
       : []),
+    ...(isEventIngestedEnabled(appContextService.getConfig())
+      ? [FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME]
+      : []),
   ];
 
   template.ignore_missing_component_templates = template.composed_of.filter(isUserSettingsTemplate);