From e221c0357eff9d311bd67b1a4b4f15e02b6cf030 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 9 Oct 2024 11:57:31 +0200 Subject: [PATCH] Add mappings for OTel event body (#114332) Also changes mappings from body_* to body.* --- .../logs-otel@mappings.yaml | 28 ++++++--- .../metrics-otel@mappings.yaml | 2 +- .../component-templates/otel@mappings.yaml | 6 +- .../semconv-resource-to-ecs@mappings.yaml | 2 +- .../src/main/resources/resources.yaml | 2 +- .../rest-api-spec/test/20_logs_tests.yml | 61 ++++++++++++++++++- 6 files changed, 85 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index 107901adb834f..5f4dcbd416720 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -14,8 +14,7 @@ template: attributes: type: passthrough dynamic: true - priority: 10 - time_series_dimension: true + priority: 20 properties: exception.type: type: keyword @@ -40,13 +39,28 @@ template: log.level: type: alias path: severity_text - body_text: - type: match_only_text + body: + type: object + properties: + text: + type: match_only_text + flattened: + # this is used for complex bodies of regular log records + # using the flattened field type avoids mapping issues which can be caused by logs containing arbitrary JSON objects + # the tradeoff is that the flattened field type is currently not supported well by Kibana and has other limitations + type: flattened + structured: + # this is used for events + # events are also represented as log records + # the event.name attribute uniquely identifies event structure / type of the payload (body) + # see also https://github.com/open-telemetry/semantic-conventions/blob/main/docs/general/events.md + # this makes them less prone to mapping issues, which is why we're enabling dynamic mappings + type: passthrough + dynamic: true + priority: 10 message: type: alias - path: body_text - body_structured: - type: flattened + path: body.text trace_id: type: keyword trace.id: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index 2f6aa7f6c916b..37dd93b7f16d9 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -10,7 +10,7 @@ template: metrics: type: passthrough dynamic: true - priority: 1 + priority: 10 unit: type: keyword time_series_dimension: true diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index fad85661203d6..513e1a857787e 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -20,7 +20,7 @@ template: attributes: type: passthrough dynamic: true - priority: 10 + priority: 20 time_series_dimension: true dropped_attributes_count: type: long @@ -39,7 +39,7 @@ template: attributes: type: passthrough dynamic: true - priority: 20 + priority: 30 time_series_dimension: true resource: properties: @@ -51,7 +51,7 @@ template: attributes: type: passthrough dynamic: true - priority: 30 + priority: 40 time_series_dimension: true dynamic_templates: - complex_attributes: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml index 87f97c7487be8..6645e7d282520 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -11,7 +11,7 @@ template: attributes: type: passthrough dynamic: true - priority: 30 + priority: 40 time_series_dimension: true properties: host.name: diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index e32037901a49c..52873287696ab 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin otel-data. This must be increased whenever an existing template is # changed, in order for it to be updated on Elasticsearch upgrade. -version: 4 +version: 5 component-templates: - otel@mappings diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index fc162d0647d08..0957a79552ad3 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -11,7 +11,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body_text":"Error: Unable to connect to the database.","severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}' + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body":{"text":"Error: Unable to connect to the database."},"severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}' - is_false: errors - do: search: @@ -39,7 +39,8 @@ setup: attributes: foo: [3, 2, 1] bar: [b, c, a] - body_text: "Error: Unable to connect to the database." + body: + text: "Error: Unable to connect to the database." severity_text: ERROR - is_false: errors - do: @@ -78,7 +79,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}' + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body": {"text":"error1"}}' - is_false: errors - do: indices.get_data_stream: @@ -90,3 +91,57 @@ setup: - is_true: $datastream-backing-index - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" } - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" } +--- +Event body: + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + resource: + attributes: + service.name: my-service + attributes: + event.name: foo + body: + structured: + foo: + bar: baz + - is_false: errors + - do: + indices.get_data_stream: + name: logs-generic.otel-default + - set: { data_streams.0.indices.0.index_name: datastream-backing-index } + - do: + indices.get_mapping: + index: $datastream-backing-index + - is_true: $datastream-backing-index + - match: { .$datastream-backing-index.mappings.properties.body.properties.structured.properties.foo\.bar.type: "keyword" } +--- +Structured log body: + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + resource: + attributes: + service.name: my-service + body: + flattened: + foo: + bar: baz + - is_false: errors + - do: + indices.get_data_stream: + name: logs-generic.otel-default + - set: { data_streams.0.indices.0.index_name: datastream-backing-index } + - do: + indices.get_mapping: + index: $datastream-backing-index + - is_true: $datastream-backing-index + - match: { .$datastream-backing-index.mappings.properties.body.properties.flattened.type: "flattened" }