From f1f6117f04ab18b3e67906b6ececae51fa5f0669 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:41:40 +0100 Subject: [PATCH] [Fleet] added `eventIngestedEnabled` flag (#199733) ## Summary Closes https://github.com/elastic/integrations/issues/11491 Added a separate flag `xpack.fleet.eventIngestedEnabled` (false by default) to keep the `event.ingested` mapping even when `agentIdVerificationEnabled` is disabled (in serverless oblt projects) Created a new pipeline `.fleet_event_ingested_pipeline-1` to use when only `eventIngestedEnabled` is enabled, to skip the step of calculating `agent_id_status`. I couldn't change `.fleet_final_pipeline-1` because the pipeline steps have to be different based on the flags. ## To verify: Note: After changing the flags, the packages have to be reinstalled to see the changes in the index templates, tested with `elastic_agent` package. Also, the data streams should be rolled over to see the changes in the ingested data. ``` POST logs-elastic_agent-default/_rollover POST logs-elastic_agent.metricbeat-default/_rollover ``` ### Default behaviour unchanged (Agent id verification enabled, event.ingested flag disabled) - by default: no change in behaviour, both `event.ingested` and `event.agent_id_status` should be mapped image image ### Agent id verification disabled, event.ingested enabled - set in `kibana.yml` ``` xpack.fleet.agentIdVerificationEnabled: false xpack.fleet.eventIngestedEnabled: true ``` - verify that `event.ingested` is mapped, `event.agent_id_status` is not image image image image image ### Agent id verification disabled, event.ingested disabled - set in `kibana.yml` ``` xpack.fleet.agentIdVerificationEnabled: false xpack.fleet.eventIngestedEnabled: false # default ``` - verify that neither `event.ingested` and `event.agent_id_status` is mapped image ### Agent id verification enabled, event.ingested enabled - set in `kibana.yml` ``` xpack.fleet.agentIdVerificationEnabled: true # default xpack.fleet.eventIngestedEnabled: true ``` - both `event.ingested` and `event.agent_id_status` should be mapped image image ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .buildkite/ftr_platform_stateful_configs.yml | 1 + config/serverless.oblt.yml | 3 + x-pack/plugins/fleet/common/types/index.ts | 1 + .../fleet/public/mock/plugin_configuration.ts | 1 + x-pack/plugins/fleet/server/config.ts | 1 + .../fleet/server/constants/fleet_es_assets.ts | 93 ++++++++- .../plugins/fleet/server/constants/index.ts | 3 + x-pack/plugins/fleet/server/mocks/index.ts | 2 + .../elasticsearch/ingest_pipeline/install.ts | 36 ++++ .../elasticsearch/template/template.test.ts | 48 ++++- .../epm/elasticsearch/template/template.ts | 12 +- x-pack/plugins/fleet/server/services/setup.ts | 6 +- .../apis/event_ingested/index.js | 17 ++ .../apis/event_ingested/use_event_ingested.ts | 197 ++++++++++++++++++ .../config.event_ingested.ts | 30 +++ .../dataset_quality/degraded_field_flyout.ts | 8 +- 16 files changed, 439 insertions(+), 20 deletions(-) create mode 100644 x-pack/test/fleet_api_integration/apis/event_ingested/index.js create mode 100644 x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts create mode 100644 x-pack/test/fleet_api_integration/config.event_ingested.ts diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index b015b1c96c73a..3db1d194e59aa 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -183,6 +183,7 @@ enabled: - x-pack/test/fleet_api_integration/config.agent.ts - x-pack/test/fleet_api_integration/config.agent_policy.ts - x-pack/test/fleet_api_integration/config.epm.ts + - x-pack/test/fleet_api_integration/config.event_ingested.ts - x-pack/test/fleet_api_integration/config.fleet.ts - x-pack/test/fleet_api_integration/config.package_policy.ts - x-pack/test/fleet_api_integration/config.space_awareness.ts diff --git a/config/serverless.oblt.yml b/config/serverless.oblt.yml index 059094ac87cdd..55e7fec7a3d39 100644 --- a/config/serverless.oblt.yml +++ b/config/serverless.oblt.yml @@ -129,6 +129,9 @@ xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability' ## Disable adding the component template `.fleet_agent_id_verification-1` to every index template for each datastream for each integration xpack.fleet.agentIdVerificationEnabled: false +## Enable event.ingested separately because agentIdVerification is disabled +xpack.fleet.eventIngestedEnabled: true + ## Enable the capability for the observability feature ID in the serverless environment to take ownership of the rules. ## The value need to be a featureId observability Or stackAlerts Or siem xpack.alerting.rules.overwriteProducer: 'observability' diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index 647a8b917d0c0..f7ce99b7f6708 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -49,6 +49,7 @@ export interface FleetConfigType { packages?: PreconfiguredPackage[]; outputs?: PreconfiguredOutput[]; agentIdVerificationEnabled?: boolean; + eventIngestedEnabled?: boolean; enableExperimental?: string[]; packageVerification?: { gpgKeyPath?: string; diff --git a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts index 935561426d7c2..30c01b1dfeb43 100644 --- a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts +++ b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts @@ -13,6 +13,7 @@ export const createConfigurationMock = (): FleetConfigType => { registryUrl: '', registryProxyUrl: '', agentIdVerificationEnabled: true, + eventIngestedEnabled: false, agents: { enabled: true, elasticsearch: { diff --git a/x-pack/plugins/fleet/server/config.ts b/x-pack/plugins/fleet/server/config.ts index ab5e06ef03716..b4f41562fd3ec 100644 --- a/x-pack/plugins/fleet/server/config.ts +++ b/x-pack/plugins/fleet/server/config.ts @@ -170,6 +170,7 @@ export const config: PluginConfigDescriptor = { proxies: PreconfiguredFleetProxiesSchema, spaceSettings: PreconfiguredSpaceSettingsSchema, agentIdVerificationEnabled: schema.boolean({ defaultValue: true }), + eventIngestedEnabled: schema.boolean({ defaultValue: false }), setup: schema.maybe( schema.object({ agentPolicySchemaUpgradeBatchSize: schema.maybe(schema.number()), diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 55e6493c77891..621adc5b3b81c 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -17,6 +17,8 @@ export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1'; export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; +export const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1'; + export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1'; export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { @@ -46,6 +48,12 @@ export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { }; export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1'; +export const INGESTED_MAPPING = { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, +}; + export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { _meta: meta, template: { @@ -58,11 +66,7 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { properties: { event: { properties: { - ingested: { - type: 'date', - format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', - ignore_malformed: false, - }, + ingested: INGESTED_MAPPING, agent_id_status: { ignore_above: 1024, type: 'keyword', @@ -74,12 +78,38 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { }, }; +export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME = '.fleet_event_ingested-1'; + +export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT = { + _meta: meta, + template: { + settings: { + index: { + final_pipeline: FLEET_EVENT_INGESTED_PIPELINE_ID, + }, + }, + mappings: { + properties: { + event: { + properties: { + ingested: INGESTED_MAPPING, + }, + }, + }, + }, + }, +}; + export const FLEET_COMPONENT_TEMPLATES = [ { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT }, { name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, }, + { + name: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + body: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT, + }, ]; export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`; @@ -96,6 +126,59 @@ export const STACK_COMPONENT_TEMPLATES = [ STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, ]; +export const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1; + +// If the content is updated you probably need to update the FLEET_EVENT_INGESTED_PIPELINE_VERSION too to allow upgrade of the pipeline +export const FLEET_EVENT_INGESTED_PIPELINE_CONTENT = `--- +version: ${FLEET_EVENT_INGESTED_PIPELINE_VERSION} +_meta: + managed_by: ${meta.managed_by} + managed: ${meta.managed} +description: > + Pipeline for processing all incoming Fleet Agent documents that adds event.ingested. +processors: + - script: + description: Add time when event was ingested (and remove sub-seconds to improve storage efficiency) + tag: truncate-subseconds-event-ingested + ignore_failure: true + source: |- + if (ctx?.event == null) { + ctx.event = [:]; + } + + ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + - remove: + description: Remove any pre-existing untrusted values. + field: + - event.agent_id_status + - _security + ignore_missing: true + - remove: + description: Remove event.original unless the preserve_original_event tag is set + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - set_security_user: + field: _security + properties: + - authentication_type + - username + - realm + - api_key + - remove: + field: _security + ignore_missing: true +on_failure: + - remove: + field: _security + ignore_missing: true + ignore_failure: true + - append: + field: error.message + value: + - 'failed in Fleet agent event_ingested_pipeline: {{ _ingest.on_failure_message }}'`; + export const FLEET_FINAL_PIPELINE_VERSION = 4; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index fb7e27c8b0ef8..48de05c0b635d 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -111,6 +111,9 @@ export { FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_ID, + FLEET_EVENT_INGESTED_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_CONTENT, FLEET_INSTALL_FORMAT_VERSION, FLEET_AGENT_POLICIES_SCHEMA_VERSION, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index f032c1f7bb8c7..8d452b394dd18 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -81,6 +81,7 @@ export const createAppContextStartContractMock = ( agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + eventIngestedEnabled: false, ...configOverrides, }; @@ -120,6 +121,7 @@ export const createAppContextStartContractMock = ( agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + eventIngestedEnabled: false, }, config$, kibanaVersion: '8.99.0', // Fake version :) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 5a4672f67fe53..51162ac2c6335 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -20,6 +20,9 @@ import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_ID, + FLEET_EVENT_INGESTED_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_CONTENT, } from '../../../../constants'; import { getPipelineNameForDatastream } from '../../../../../common/services'; import type { ArchiveEntry, PackageInstallContext } from '../../../../../common/types'; @@ -302,6 +305,39 @@ export async function ensureFleetFinalPipelineIsInstalled( return { isCreated: false }; } +export async function ensureFleetEventIngestedPipelineIsInstalled( + esClient: ElasticsearchClient, + logger: Logger +) { + const esClientRequestOptions: TransportRequestOptions = { + ignore: [404], + }; + const res = await esClient.ingest.getPipeline( + { id: FLEET_EVENT_INGESTED_PIPELINE_ID }, + { ...esClientRequestOptions, meta: true } + ); + + const installedVersion = res?.body[FLEET_EVENT_INGESTED_PIPELINE_ID]?.version; + if ( + res.statusCode === 404 || + !installedVersion || + installedVersion < FLEET_EVENT_INGESTED_PIPELINE_VERSION + ) { + await installPipeline({ + esClient, + logger, + pipeline: { + nameForInstallation: FLEET_EVENT_INGESTED_PIPELINE_ID, + contentForInstallation: FLEET_EVENT_INGESTED_PIPELINE_CONTENT, + extension: 'yml', + }, + }); + return { isCreated: true }; + } + + return { isCreated: false }; +} + const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/'); const isDataStreamPipeline = (path: string, dataStreamDataset: string) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index c7d2e4eacb32a..c06d0cdbb6429 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -14,7 +14,11 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { errors } from '@elastic/elasticsearch'; -import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets'; +import { + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, +} from '../../../../constants/fleet_es_assets'; import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; @@ -22,7 +26,6 @@ import type { RegistryDataStream } from '../../../../types'; import { processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { - FLEET_COMPONENT_TEMPLATES, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, @@ -36,10 +39,6 @@ import { updateCurrentWriteIndices, } from './template'; -const FLEET_COMPONENT_TEMPLATES_NAMES = FLEET_COMPONENT_TEMPLATES.map( - (componentTemplate) => componentTemplate.name -); - // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ print(val) { @@ -88,7 +87,8 @@ describe('EPM template', () => { STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, ...composedOfTemplates, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); @@ -108,7 +108,8 @@ describe('EPM template', () => { 'metrics@tsdb-settings', ...composedOfTemplates, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); @@ -138,6 +139,34 @@ describe('EPM template', () => { ]); }); + it('creates fleet event ingested component template if event ingested flag is enabled', () => { + appContextService.start( + createAppContextStartContractMock({ + agentIdVerificationEnabled: false, + eventIngestedEnabled: true, + }) + ); + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + templateIndexPattern: 'logs-*', + type: 'logs', + packageName: 'nginx', + composedOfTemplates, + templatePriority: 200, + mappings: { properties: [] }, + isIndexModeTimeSeries: false, + }); + expect(template.composed_of).toStrictEqual([ + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + ...composedOfTemplates, + STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + ]); + }); + it('adds empty composed_of correctly', () => { const composedOfTemplates: string[] = []; @@ -154,7 +183,8 @@ describe('EPM template', () => { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3709975c57a5e..b9c0846f3e4f2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -15,7 +15,10 @@ import type { import pMap from 'p-map'; import { isResponseError } from '@kbn/es-errors'; -import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets'; +import { + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, +} from '../../../../constants/fleet_es_assets'; import type { Field, Fields } from '../../fields/field'; import type { @@ -27,6 +30,7 @@ import type { } from '../../../../types'; import { appContextService } from '../../..'; import { getRegistryDataStreamAssetBaseName } from '../../../../../common/services'; +import type { FleetConfigType } from '../../../../../common/types'; import { STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, @@ -115,6 +119,9 @@ export function getTemplate({ const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries); + const isEventIngestedEnabled = (config?: FleetConfigType): boolean => + Boolean(!config?.agentIdVerificationEnabled && config?.eventIngestedEnabled); + template.composed_of = [ ...esBaseComponents, ...(template.composed_of || []), @@ -123,6 +130,9 @@ export function getTemplate({ ...(appContextService.getConfig()?.agentIdVerificationEnabled ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME] : []), + ...(isEventIngestedEnabled(appContextService.getConfig()) + ? [FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME] + : []), ]; template.ignore_missing_component_templates = template.composed_of.filter(isUserSettingsTemplate); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 0d6ec183531a4..ab882a013ebe3 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -36,7 +36,10 @@ import { downloadSourceService } from './download_source'; import { getRegistryUrl, settingsService } from '.'; import { awaitIfPending } from './setup_utils'; -import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; +import { + ensureFleetEventIngestedPipelineIsInstalled, + ensureFleetFinalPipelineIsInstalled, +} from './epm/elasticsearch/ingest_pipeline/install'; import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, reinstallPackageForInstallation } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; @@ -336,6 +339,7 @@ export async function ensureFleetGlobalEsAssets( const globalAssetsRes = await Promise.all([ ensureDefaultComponentTemplates(esClient, logger), // returns an array ensureFleetFinalPipelineIsInstalled(esClient, logger), + ensureFleetEventIngestedPipelineIsInstalled(esClient, logger), ]); const assetResults = globalAssetsRes.flat(); if (assetResults.some((asset) => asset.isCreated)) { diff --git a/x-pack/test/fleet_api_integration/apis/event_ingested/index.js b/x-pack/test/fleet_api_integration/apis/event_ingested/index.js new file mode 100644 index 0000000000000..c6c76ca423b5f --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/event_ingested/index.js @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setupTestUsers } from '../test_users'; + +export default function loadTests({ loadTestFile, getService }) { + describe('Event Ingested', () => { + before(async () => { + await setupTestUsers(getService('security')); + }); + loadTestFile(require.resolve('./use_event_ingested')); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts b/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts new file mode 100644 index 0000000000000..7badbedbd77ba --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { testUsers } from '../test_users'; + +const TEST_INDEX = 'logs-log.log-test'; + +const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1'; + +// TODO: Use test package or move to input package version github.com/elastic/kibana/issues/154243 +const LOG_INTEGRATION_VERSION = '1.1.2'; + +const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const es = getService('es'); + const esArchiver = getService('esArchiver'); + const fleetAndAgents = getService('fleetAndAgents'); + + describe('fleet_event_ingested_pipeline', () => { + skipIfNoDockerRegistry(providerContext); + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + await fleetAndAgents.setup(); + // Use the custom log package to test the fleet final pipeline + await supertestWithoutAuth + .post(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + }); + + after(async () => { + await supertestWithoutAuth + .delete(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + const res = await es.search({ + index: TEST_INDEX, + }); + + for (const hit of res.hits.hits) { + await es.delete({ + id: hit._id!, + index: hit._index, + }); + } + }); + + it('should correctly update the event ingested pipeline', async () => { + await es.ingest.putPipeline({ + id: FLEET_EVENT_INGESTED_PIPELINE_ID, + body: { + description: 'Test PIPELINE WITHOUT version', + processors: [ + { + set: { + field: 'my-keyword-field', + value: 'foo', + }, + }, + ], + }, + }); + await supertestWithoutAuth + .post(`/api/fleet/setup`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx'); + const pipelineRes = await es.ingest.getPipeline({ id: FLEET_EVENT_INGESTED_PIPELINE_ID }); + expect(pipelineRes).to.have.property(FLEET_EVENT_INGESTED_PIPELINE_ID); + expect(pipelineRes[FLEET_EVENT_INGESTED_PIPELINE_ID].version).to.be(1); + }); + + it('should correctly setup the event ingested pipeline and apply to fleet managed index template', async () => { + const pipelineRes = await es.ingest.getPipeline({ id: FLEET_EVENT_INGESTED_PIPELINE_ID }); + expect(pipelineRes).to.have.property(FLEET_EVENT_INGESTED_PIPELINE_ID); + const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); + expect(res.index_templates.length).to.be(FLEET_EVENT_INGESTED_PIPELINE_VERSION); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('ecs@mappings'); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('.fleet_globals-1'); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain( + '.fleet_event_ingested-1' + ); + }); + + it('all docs should contain event.ingested without sub-seconds', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + '@timestamp': '2020-01-01T09:09:00', + message: 'hello', + }, + }); + + const doc = await es.get({ + id: res._id, + index: res._index, + }); + // @ts-expect-error + const ingestTimestamp = doc._source.event.ingested; + + // 2021-06-30T12:06:28Z + expect(ingestTimestamp).to.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/); + }); + + it('should remove agent_id_status', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + '@timestamp': '2020-01-01T09:09:00', + agent: { + id: 'agent1', + }, + event: { + agent_id_status: 'dummy', + }, + }, + }); + + const doc = await es.get({ + id: res._id, + index: res._index, + }); + // @ts-expect-error + const event = doc._source.event; + + expect(event.agent_id_status).to.be(undefined); + expect(event).to.have.property('ingested'); + }); + + it('removes event.original if preserve_original_event is not set', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + event: { + original: JSON.stringify({ foo: 'bar' }), + }, + '@timestamp': '2023-01-01T09:00:00', + tags: [], + agent: { + id: 'agent1', + }, + }, + }); + + const doc: any = await es.get({ + id: res._id, + index: res._index, + }); + + const event = doc._source.event; + + expect(event.original).to.be(undefined); + }); + + it('preserves event.original if preserve_original_event is set', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + event: { + original: JSON.stringify({ foo: 'bar' }), + }, + '@timestamp': '2023-01-01T09:00:00', + tags: ['preserve_original_event'], + agent: { + id: 'agent1', + }, + }, + }); + + const doc: any = await es.get({ + id: res._id, + index: res._index, + }); + + const event = doc._source.event; + + expect(event.original).to.eql(JSON.stringify({ foo: 'bar' })); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/config.event_ingested.ts b/x-pack/test/fleet_api_integration/config.event_ingested.ts new file mode 100644 index 0000000000000..cbdf4d501e1d2 --- /dev/null +++ b/x-pack/test/fleet_api_integration/config.event_ingested.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts')); + const serverArgs: string[] = [ + ...baseFleetApiConfig.get('kbnTestServer.serverArgs'), + // serverless oblt needs only event.ingested, without agent id verification + `--xpack.fleet.agentIdVerificationEnabled=false`, + `--xpack.fleet.eventIngestedEnabled=true`, + ]; + + return { + ...baseFleetApiConfig.getAll(), + kbnTestServer: { + ...baseFleetApiConfig.get('kbnTestServer'), + serverArgs, + }, + testFiles: [require.resolve('./apis/event_ingested')], + junit: { + reportName: 'X-Pack Event Ingested API Integration Tests', + }, + }; +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 9fb1f74cbae0e..42a5a095ed6c4 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -194,7 +194,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Set Limit of 42 await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { - 'mapping.total_fields.limit': 43, + 'mapping.total_fields.limit': 42, }); await synthtrace.index([ @@ -262,13 +262,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Set Limit of 44 + // Set Limit of 43 await PageObjects.datasetQuality.setDataStreamSettings( PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ dataset: nginxAccessDatasetName, }) + '-000002', { - 'mapping.total_fields.limit': 44, + 'mapping.total_fields.limit': 43, } ); @@ -745,7 +745,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'disabled' ); - expect(currentFieldLimit).to.be(44); + expect(currentFieldLimit).to.be(43); expect(currentFieldLimitDisabledStatus).to.be('true'); // Should display new field limit