From 2f65826e7b2aeab9159b624f996c5fa963dcc297 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Mon, 20 May 2024 22:41:20 +0100 Subject: [PATCH] Update apm-data and remap for OTel hostmetrics to elastic metrics --- .gitignore | 3 ++- changelogs/head.asciidoc | 2 ++ go.mod | 4 +++- go.sum | 8 ++++++-- internal/beater/otlp/grpc.go | 7 ++++--- internal/beater/otlp/http.go | 7 ++++--- 6 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index c7bfbb2ebc1..b95de3a4d30 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ version.asciidoc.bck .imported-java-agent-pubkey /agent go.work +go.work.* testing/docker/metricbeat/apm-server.yml cmd/intake-receiver/intake-receiver cmd/intake-receiver/events @@ -50,4 +51,4 @@ testing/smoke/**/main.tf !testing/smoke/supported-os/main.tf !testing/smoke/managed/main.tf testing/rally-cloud/build -.bck \ No newline at end of file +.bck diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 1449a322f50..4c3d4b5c772 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -8,6 +8,7 @@ https://github.com/elastic/apm-server/compare/8.14\...main[View commits] - Avoid data race due to reuse of `bytes.Buffer` in ES bulk requests {pull}13155[13155] - APM Server now relies on the Elasticsearch apm-data plugin's index templates, which reverts some unsafe uses of `flattened` field types {pull}12066[12066] +- Add `error.id` to jaeger errors {pull}13196[13196] [float] ==== Breaking Changes @@ -23,3 +24,4 @@ https://github.com/elastic/apm-server/compare/8.14\...main[View commits] - Add `elasticsearch.flushed.uncompressed.bytes` metric {pull}13155[13155] - APM Server now relies on the Elasticsearch apm-data plugin's index templates, removing the requirement to install the APM integration package {pull}12066[12066] +- Enable Kibana curated UIs to work with hostmetrics from OpenTelemetry's [hostmetricsreceiver](https://pkg.go.dev/go.opentelemetry.io/collector/receiver/hostmetricsreceiver) {pull}13196[13196] diff --git a/go.mod b/go.mod index a446c91dafa..f7c94ed8356 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 - github.com/elastic/apm-data v1.1.0 + github.com/elastic/apm-data v1.2.0 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240604124742-217f5a626498 github.com/elastic/elastic-agent-client/v7 v7.10.0 github.com/elastic/elastic-agent-libs v0.9.11 @@ -92,6 +92,7 @@ require ( github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.14.3 // indirect + github.com/elastic/opentelemetry-lib v0.0.0-20240604140721-08863a456d6c // indirect github.com/elastic/pkcs8 v1.0.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/fatih/color v1.15.0 // indirect @@ -127,6 +128,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.102.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/planetscale/vtprotobuf v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/prometheus/client_golang v1.19.1 // indirect diff --git a/go.sum b/go.sum index 5f5acfcd655..f109bbd66b0 100644 --- a/go.sum +++ b/go.sum @@ -145,8 +145,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HEE0a5BDFmhEmJV/Uo4uzj/paHjd7yl6+KYguw= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= -github.com/elastic/apm-data v1.1.0 h1:5ahaTQwWb6+OmrXrc1dzfD2wjvTG7yJRaldcsc9dIc4= -github.com/elastic/apm-data v1.1.0/go.mod h1:TLHPd2H8wYlf3FIqriQ018NZzE1FLC6VtsQ6DN0vRmY= +github.com/elastic/apm-data v1.2.0 h1:yRFbpDywG5uHEpFByPqpMilEBdcF+5xs5RkOjFJD1h8= +github.com/elastic/apm-data v1.2.0/go.mod h1:DPI9Fup+vDWpVbGNRGtObCH9uikOc3jYNDJY+i0+e78= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240604124742-217f5a626498 h1:cbJwEc0mNyTUhOQc5Ap6Z3rQoHyZw+SmaIyWDCupB3Q= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240604124742-217f5a626498/go.mod h1:7KXJ/qUgiBX1mZkHRojpy3h8+wgMgug09qW9Lj4I4To= github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP+WioV73ia17Et+UlFbz8= @@ -177,6 +177,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/opentelemetry-lib v0.0.0-20240604140721-08863a456d6c h1:gRCjiqyIH1RxEYAJr1z9Y3KdJITip45iPprj7DbL9Mk= +github.com/elastic/opentelemetry-lib v0.0.0-20240604140721-08863a456d6c/go.mod h1:/kKvHbJLVo/NcKMPHI8/RZKL64fushmnRUzn+arQpjg= github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 h1:5/RUNg7rkIvayjPhAIoI3v8p45NfWcfWs5DZSElycis= @@ -438,6 +440,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA= +github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/internal/beater/otlp/grpc.go b/internal/beater/otlp/grpc.go index 82722d1c92d..2684e1bd398 100644 --- a/internal/beater/otlp/grpc.go +++ b/internal/beater/otlp/grpc.go @@ -74,9 +74,10 @@ func RegisterGRPCServices( // at any time, and instead aggregate metrics from consumers that are // dynamically registered and unregistered. consumer := otlp.NewConsumer(otlp.ConsumerConfig{ - Processor: processor, - Logger: logger, - Semaphore: semaphore, + Processor: processor, + Logger: logger, + Semaphore: semaphore, + RemapOTelMetrics: true, }) gRPCMonitoredConsumer.set(consumer) diff --git a/internal/beater/otlp/http.go b/internal/beater/otlp/http.go index aaf9a075066..5600ea5dd62 100644 --- a/internal/beater/otlp/http.go +++ b/internal/beater/otlp/http.go @@ -57,9 +57,10 @@ func NewHTTPHandlers(logger *zap.Logger, processor modelpb.BatchProcessor, semap // at any time, and instead aggregate metrics from consumers that are // dynamically registered and unregistered. consumer := otlp.NewConsumer(otlp.ConsumerConfig{ - Processor: processor, - Logger: logger, - Semaphore: semaphore, + Processor: processor, + Logger: logger, + Semaphore: semaphore, + RemapOTelMetrics: true, }) httpMonitoredConsumer.set(consumer) return HTTPHandlers{consumer: consumer}