Update apm-data and remap for OTel hostmetrics to elastic metrics

.gitignore

@@ -33,6 +33,7 @@ version.asciidoc.bck
 .imported-java-agent-pubkey
 /agent
 go.work
+go.work.*
 testing/docker/metricbeat/apm-server.yml
 cmd/intake-receiver/intake-receiver
 cmd/intake-receiver/events
@@ -50,4 +51,4 @@ testing/smoke/**/main.tf
 !testing/smoke/supported-os/main.tf
 !testing/smoke/managed/main.tf
 testing/rally-cloud/build
-.bck
+.bck

changelogs/head.asciidoc

@@ -8,6 +8,7 @@ https://github.com/elastic/apm-server/compare/8.14\...main[View commits]
 
 - Avoid data race due to reuse of `bytes.Buffer` in ES bulk requests {pull}13155[13155]
 - APM Server now relies on the Elasticsearch apm-data plugin's index templates, which reverts some unsafe uses of `flattened` field types {pull}12066[12066]
+- Add `error.id` to jaeger errors {pull}13196[13196]
 
 [float]
 ==== Breaking Changes
@@ -23,3 +24,4 @@ https://github.com/elastic/apm-server/compare/8.14\...main[View commits]
 
 - Add `elasticsearch.flushed.uncompressed.bytes` metric {pull}13155[13155]
 - APM Server now relies on the Elasticsearch apm-data plugin's index templates, removing the requirement to install the APM integration package {pull}12066[12066]
+- Enable Kibana curated UIs to work with hostmetrics from OpenTelemetry's [hostmetricsreceiver](https://pkg.go.dev/go.opentelemetry.io/collector/receiver/hostmetricsreceiver) {pull}13196[13196]

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.4
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6
-	github.com/elastic/apm-data v1.1.0
+	github.com/elastic/apm-data v1.2.0
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240604124742-217f5a626498
 	github.com/elastic/elastic-agent-client/v7 v7.10.0
 	github.com/elastic/elastic-agent-libs v0.9.11
@@ -92,6 +92,7 @@ require (
 	github.com/elastic/go-structform v0.0.10 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
+	github.com/elastic/opentelemetry-lib v0.0.0-20240604140721-08863a456d6c // indirect
 	github.com/elastic/pkcs8 v1.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
@@ -127,6 +128,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.102.0 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/planetscale/vtprotobuf v0.6.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect

go.sum

@@ -145,8 +145,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HEE0a5BDFmhEmJV/Uo4uzj/paHjd7yl6+KYguw=
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo=
-github.com/elastic/apm-data v1.1.0 h1:5ahaTQwWb6+OmrXrc1dzfD2wjvTG7yJRaldcsc9dIc4=
-github.com/elastic/apm-data v1.1.0/go.mod h1:TLHPd2H8wYlf3FIqriQ018NZzE1FLC6VtsQ6DN0vRmY=
+github.com/elastic/apm-data v1.2.0 h1:yRFbpDywG5uHEpFByPqpMilEBdcF+5xs5RkOjFJD1h8=
+github.com/elastic/apm-data v1.2.0/go.mod h1:DPI9Fup+vDWpVbGNRGtObCH9uikOc3jYNDJY+i0+e78=
 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240604124742-217f5a626498 h1:cbJwEc0mNyTUhOQc5Ap6Z3rQoHyZw+SmaIyWDCupB3Q=
 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240604124742-217f5a626498/go.mod h1:7KXJ/qUgiBX1mZkHRojpy3h8+wgMgug09qW9Lj4I4To=
 github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP+WioV73ia17Et+UlFbz8=
@@ -177,6 +177,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
 github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
 github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/opentelemetry-lib v0.0.0-20240604140721-08863a456d6c h1:gRCjiqyIH1RxEYAJr1z9Y3KdJITip45iPprj7DbL9Mk=
+github.com/elastic/opentelemetry-lib v0.0.0-20240604140721-08863a456d6c/go.mod h1:/kKvHbJLVo/NcKMPHI8/RZKL64fushmnRUzn+arQpjg=
 github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA=
 github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0=
 github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 h1:5/RUNg7rkIvayjPhAIoI3v8p45NfWcfWs5DZSElycis=
@@ -438,6 +440,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

internal/beater/otlp/grpc.go

@@ -74,9 +74,10 @@ func RegisterGRPCServices(
 	// at any time, and instead aggregate metrics from consumers that are
 	// dynamically registered and unregistered.
 	consumer := otlp.NewConsumer(otlp.ConsumerConfig{
-		Processor: processor,
-		Logger:    logger,
-		Semaphore: semaphore,
+		Processor:        processor,
+		Logger:           logger,
+		Semaphore:        semaphore,
+		RemapOTelMetrics: true,
 	})
 	gRPCMonitoredConsumer.set(consumer)
 

internal/beater/otlp/http.go

@@ -57,9 +57,10 @@ func NewHTTPHandlers(logger *zap.Logger, processor modelpb.BatchProcessor, semap
 	// at any time, and instead aggregate metrics from consumers that are
 	// dynamically registered and unregistered.
 	consumer := otlp.NewConsumer(otlp.ConsumerConfig{
-		Processor: processor,
-		Logger:    logger,
-		Semaphore: semaphore,
+		Processor:        processor,
+		Logger:           logger,
+		Semaphore:        semaphore,
+		RemapOTelMetrics: true,
 	})
 	httpMonitoredConsumer.set(consumer)
 	return HTTPHandlers{consumer: consumer}