Skip to content

Commit

Permalink
[Fleet] Avoid subobject/scalar mapping conflicts by setting subobject…
Browse files Browse the repository at this point in the history 
…s: false on custom integrations (#178397)

In case of a property in indexed documents is sometimes a scalar value
(e.g. a number or string) and sometimes an object with sub-properties,
mapping conflicts will prevent data from being indexed properly.

The `subobjects` mapping property makes sure no additional subobject
mappings will be defined during runtime. You can learn more here:
https://www.elastic.co/guide/en/elasticsearch/reference/current/subobjects.html

This PR sets `subobjects: false` on logs data streams for custom
integrations to avoid. This only affects newly created integrations.

The setting is set via the `@package` component template.

This only requires a small change in the generated datastream entry, as
`index_template.mappings` are already spread into the index config of
the component template:
https://github.com/elastic/kibana/blob/c98ee2f6c10992f372ebea4ce7d91f02a68fce5c/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts#L380
  • Loading branch information
@flash1293
flash1293 authored Mar 14, 2024
1 parent f0239e1 commit 98aff21
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 202 deletions.
8 changes: 8 additions & 0 deletions ...ck/plugins/fleet/server/services/epm/packages/custom_integrations/assets/dataset/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* 2.0.
*/

import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';

import type { CustomPackageDatasetConfiguration } from '../../../install';

export const generateDatastreamEntries = (
Expand All @@ -22,6 +24,12 @@ export const generateDatastreamEntries = (
release: 'ga' as const,
// NOTE: This ensures our default.yml pipeline is used as the default_pipeline in the index template
ingest_pipeline: 'default',
elasticsearch: {
// TODO: Needs to be cast because https://github.com/elastic/elasticsearch-specification/pull/2445 hasn't landed yet, can be removed once it has
'index_template.mappings': {
subobjects: false,
} as MappingTypeMapping,
},
};
});
};
345 changes: 143 additions & 202 deletions x-pack/test/fleet_api_integration/apis/epm/install_custom.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ export default function (providerContext: FtrProviderContext) {
const response = await esClient.indices.getMapping({ index: indexName });

expect(Object.values(response)[0].mappings).to.eql({
subobjects: false,
_meta: {
managed_by: 'fleet',
managed: true,
Expand Down Expand Up @@ -343,225 +344,165 @@ export default function (providerContext: FtrProviderContext) {
type: 'date',
ignore_malformed: false,
},
cloud: {
properties: {
account: {
properties: {
id: {
type: 'keyword',
ignore_above: 1024,
},
},
},
availability_zone: {
type: 'keyword',
ignore_above: 1024,
},
image: {
properties: {
id: {
type: 'keyword',
ignore_above: 1024,
},
},
},
instance: {
properties: {
id: {
type: 'keyword',
ignore_above: 1024,
},
name: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
},
},
machine: {
properties: {
type: {
type: 'keyword',
ignore_above: 1024,
},
},
},
project: {
properties: {
id: {
type: 'keyword',
ignore_above: 1024,
},
},
},
provider: {
type: 'keyword',
ignore_above: 1024,
},
region: {
type: 'keyword',
ignore_above: 1024,
},
},
'cloud.account.id': {
type: 'keyword',
ignore_above: 1024,
},
container: {
properties: {
id: {
type: 'keyword',
ignore_above: 1024,
},
image: {
properties: {
name: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
},
},
labels: {
properties: {
foo_id: {
type: 'keyword',
ignore_above: 1024,
},
},
},
name: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
},
'cloud.availability_zone': {
type: 'keyword',
ignore_above: 1024,
},
data_stream: {
properties: {
dataset: {
type: 'constant_keyword',
},
namespace: {
type: 'constant_keyword',
},
type: {
type: 'constant_keyword',
value: 'logs',
'cloud.image.id': {
type: 'keyword',
ignore_above: 1024,
},
'cloud.instance.id': {
type: 'keyword',
ignore_above: 1024,
},
'cloud.instance.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
event: {
properties: {
agent_id_status: {
type: 'keyword',
ignore_above: 1024,
},
ingested: {
type: 'date',
format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
'cloud.machine.type': {
type: 'keyword',
ignore_above: 1024,
},
'cloud.project.id': {
type: 'keyword',
ignore_above: 1024,
},
'cloud.provider': {
type: 'keyword',
ignore_above: 1024,
},
'cloud.region': {
type: 'keyword',
ignore_above: 1024,
},
'container.id': {
type: 'keyword',
ignore_above: 1024,
},
'container.image.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
host: {
properties: {
architecture: {
type: 'keyword',
ignore_above: 1024,
},
domain: {
type: 'keyword',
ignore_above: 1024,
},
hostname: {
type: 'keyword',
ignore_above: 1024,
},
mac: {
type: 'keyword',
ignore_above: 1024,
},
id: {
type: 'keyword',
ignore_above: 1024,
},
ip: {
type: 'ip',
},
name: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
os: {
properties: {
build: {
type: 'keyword',
ignore_above: 1024,
},
codename: {
type: 'keyword',
ignore_above: 1024,
},
containerized: {
type: 'boolean',
},
family: {
type: 'keyword',
ignore_above: 1024,
},
kernel: {
type: 'keyword',
ignore_above: 1024,
},
name: {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
platform: {
type: 'keyword',
ignore_above: 1024,
},
type: {
type: 'keyword',
ignore_above: 1024,
},
version: {
type: 'keyword',
ignore_above: 1024,
},
},
'container.labels.foo_id': {
type: 'keyword',
ignore_above: 1024,
},
'container.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
input: {
properties: {
type: {
type: 'keyword',
ignore_above: 1024,
'data_stream.dataset': {
type: 'constant_keyword',
},
'data_stream.namespace': {
type: 'constant_keyword',
},
'data_stream.type': {
type: 'constant_keyword',
value: 'logs',
},
'event.agent_id_status': {
type: 'keyword',
ignore_above: 1024,
},
'event.ingested': {
type: 'date',
format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
},
'host.architecture': {
type: 'keyword',
ignore_above: 1024,
},
'host.domain': {
type: 'keyword',
ignore_above: 1024,
},
'host.hostname': {
type: 'keyword',
ignore_above: 1024,
},
'host.id': {
type: 'keyword',
ignore_above: 1024,
},
'host.ip': {
type: 'ip',
},
'host.mac': {
type: 'keyword',
ignore_above: 1024,
},
'host.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
log: {
properties: {
offset: {
type: 'long',
'host.os.build': {
type: 'keyword',
ignore_above: 1024,
},
'host.os.codename': {
type: 'keyword',
ignore_above: 1024,
},
'host.os.containerized': {
type: 'boolean',
},
'host.os.family': {
type: 'keyword',
ignore_above: 1024,
},
'host.os.kernel': {
type: 'keyword',
ignore_above: 1024,
},
'host.os.name': {
type: 'keyword',
fields: {
text: {
type: 'match_only_text',
},
},
},
'host.os.platform': {
type: 'keyword',
ignore_above: 1024,
},
'host.os.type': {
type: 'keyword',
ignore_above: 1024,
},
'host.os.version': {
type: 'keyword',
ignore_above: 1024,
},
'input.type': {
type: 'keyword',
ignore_above: 1024,
},
'log.offset': {
type: 'long',
},
},
});
});
Expand Down

0 comments on commit 98aff21

Please sign in to comment.