From 17406c2b01970b712ceff100db981ccd768dcd60 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:28:42 -0400 Subject: [PATCH 01/16] chore: Update to elastic/beats@d3c42ef75b93 (#11889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 343fe82bc87..06488552310 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20231016183702-4c34c41039be +Version: v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231016183702-4c34c41039be/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231018064857-d3c42ef75b93/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of diff --git a/go.mod b/go.mod index d705492cb8c..a879d3d7a62 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231016183702-4c34c41039be + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 github.com/elastic/elastic-agent-client/v7 v7.4.0 github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b github.com/elastic/elastic-agent-system-metrics v0.7.0 diff --git a/go.sum b/go.sum index 44bdfffe376..e3077e6c7ca 100644 --- a/go.sum +++ b/go.sum @@ -136,10 +136,10 @@ github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HE github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7 h1:QcfCHAh1bHj9WN6y75bD6mBlIIlz9dJVKJSJVXCu4TM= github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231016183702-4c34c41039be h1:zLJ2EjxgMasoIBqkYYpizEzR5yno9lJp2YDgIZydNpM= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231016183702-4c34c41039be/go.mod h1:3oRWKLGbtO3OZD/tvh9i9j2tRjwWznlg/iqs1/quMHw= -github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+PiWyZ4I4F4LqrCYgYlNc= -github.com/elastic/elastic-agent-autodiscover v0.6.2/go.mod h1:yXYKFAG+Py+TcE4CCR8EAbJiYb+6Dz9sCDoWgOveqtU= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 h1:Jkk/Vq79nJVDo4OtqFR/8TuzLqB9Zkv1sq2cyYceCm4= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93/go.mod h1:nVHoLIj1V0g6XdScm2ZbIezfuf3FKCHWTaFZ5DnPfSU= +github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA= +github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM= From 7e4babf9b4ed09f3d7a10333343bfec9ca8c0828 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 19 Oct 2023 07:12:07 +0800 Subject: [PATCH 02/16] docs: Fix Go distributed tracing example (#11854) * docs: Fix Go distributed tracing example * Link to go agent v2 docs --------- Co-authored-by: Brandon Morelli --- docs/tab-widgets/distributed-trace-receive.asciidoc | 6 +++--- docs/tab-widgets/distributed-trace-send.asciidoc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/tab-widgets/distributed-trace-receive.asciidoc b/docs/tab-widgets/distributed-trace-receive.asciidoc index 60dcbd1dd11..ecbeebfaa1f 100644 --- a/docs/tab-widgets/distributed-trace-receive.asciidoc +++ b/docs/tab-widgets/distributed-trace-receive.asciidoc @@ -3,8 +3,8 @@ // Need help with this example 1. Parse the incoming `TraceContext` with -https://godoc.org/go.elastic.co/apm/module/apmhttp#ParseTraceparentHeader[`ParseTraceparentHeader`] or -https://godoc.org/go.elastic.co/apm/module/apmhttp#ParseTracestateHeader[`ParseTracestateHeader`]. +https://pkg.go.dev/go.elastic.co/apm/module/apmhttp/v2#ParseTraceparentHeader[`ParseTraceparentHeader`] or +https://pkg.go.dev/go.elastic.co/apm/module/apmhttp/v2#ParseTracestateHeader[`ParseTracestateHeader`]. 2. Start a new transaction or span as a child of the incoming transaction with {apm-go-ref}/api.html#tracer-api-start-transaction-options[`StartTransactionOptions`] or @@ -21,7 +21,7 @@ traceContext.State, _ = apmhttp.ParseTracestateHeader(r.Header["Tracestate"]...) opts := apm.TransactionOptions{ TraceContext: traceContext, <3> } -transaction := apm.DefaultTracer.StartTransactionOptions("GET /", "request", opts) <4> +transaction := apm.DefaultTracer().StartTransactionOptions("GET /", "request", opts) <4> ---- <1> Parse the `TraceParent` header <2> Parse the `Tracestate` header diff --git a/docs/tab-widgets/distributed-trace-send.asciidoc b/docs/tab-widgets/distributed-trace-send.asciidoc index af3a689f6b0..b38b179f4f0 100644 --- a/docs/tab-widgets/distributed-trace-send.asciidoc +++ b/docs/tab-widgets/distributed-trace-send.asciidoc @@ -12,11 +12,11 @@ Example: [source,go] ---- -transaction := apm.DefaultTracer.StartTransaction("GET /", "request") <1> +transaction := apm.DefaultTracer().StartTransaction("GET /", "request") <1> traceContext := transaction.TraceContext() <2> // Send TraceContext to receiving service -traceparent := apmhttp.FormatTraceparentHeader(traceContext)) <3> +traceparent := apmhttp.FormatTraceparentHeader(traceContext) <3> tracestate := traceContext.State.String() ---- <1> Start a transaction From 00fb30eb6fe1b5c41f7e05d7b2a7e6f484cfd4cc Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 19 Oct 2023 18:25:25 +0800 Subject: [PATCH 03/16] docs: Document privileges to read RUM source maps; Update examples (#11741) * Document required privileges to read RUM source map from ES since 8.7. * Update API key creation examples, add agentcfg role to it. --- docs/access-api-keys.asciidoc | 38 ++++++++++++++++++++++++++++------- docs/configure/rum.asciidoc | 2 +- docs/feature-roles.asciidoc | 37 ++++++++++++++++++++++++++++++---- 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/docs/access-api-keys.asciidoc b/docs/access-api-keys.asciidoc index 8093843c674..fa5b9fe8c77 100644 --- a/docs/access-api-keys.asciidoc +++ b/docs/access-api-keys.asciidoc @@ -34,12 +34,24 @@ In the role descriptors box, assign the appropriate privileges to the new API ke { "names": ["{beat_default_index_prefix}-*"], "privileges": ["create_index", "create_doc"] - }, + } + ] + }, + "{beat_default_index_prefix}_sourcemap": { + "index": [ { - "names": ["{beat_default_index_prefix}-*sourcemap"], + "names": [".apm-source-map"], "privileges": ["read"] - }, - ] + } + ] + }, + "{beat_default_index_prefix}_agentcfg": { + "index": [ + { + "names": [".apm-agent-configuration"], + "privileges": ["read"] + } + ] } } ---- @@ -123,11 +135,23 @@ POST /_security/api_key { "names": ["{beat_default_index_prefix}-*"], "privileges": ["create_index", "create_doc"] - }, + } + ] + }, + "{beat_default_index_prefix}_sourcemap": { + "index": [ + { + "names": [".apm-source-map"], + "privileges": ["read"] + } + ] + }, + "{beat_default_index_prefix}_agentcfg": { + "index": [ { - "names": ["{beat_default_index_prefix}-*sourcemap"], + "names": [".apm-agent-configuration"], "privileges": ["read"] - }, + } ] } } diff --git a/docs/configure/rum.asciidoc b/docs/configure/rum.asciidoc index 7d8cc7b59ae..2bc25ec9621 100644 --- a/docs/configure/rum.asciidoc +++ b/docs/configure/rum.asciidoc @@ -134,7 +134,7 @@ the options in this section are only supported by the APM Server binary. == `source_mapping.enabled` Used to enable/disable <> for RUM events. When enabled, the APM Server needs additional privileges to read source maps. -See <> for more details. +See <> for more details. Default: `true` diff --git a/docs/feature-roles.asciidoc b/docs/feature-roles.asciidoc index 65a801ce8b2..d99e70f1cf1 100644 --- a/docs/feature-roles.asciidoc +++ b/docs/feature-roles.asciidoc @@ -45,6 +45,7 @@ information, and another for viewing it. * <>: To create and manage API keys. * <>: To view APM Agent central configurations. +* <>: To read RUM source maps. {es-security-features} provides {ref}/built-in-roles.html[built-in roles] that grant a subset of the privileges needed by APM users. @@ -91,12 +92,10 @@ that has the following privileges: |Write events into {es} |==== -. If <> is enabled, additional privileges are required to read source maps. -See {kibana-ref}/rum-sourcemap-api.html[RUM source map API] for more details. -Assign these extra privileges to the *general writer role*. - . Assign the *general writer role* to users who need to publish APM data. +. If <> is enabled, create a separate <>. + //// *********************************** *********************************** *********************************** *********************************** @@ -354,6 +353,36 @@ which requires the following privileges: TIP: Looking for privileges and roles needed to use central configuration from the {apm-app} or {apm-app} API? See {kibana-ref}/apm-app-central-config-user.html[{apm-app} central configuration user]. +[[privileges-rum-source-map]] +=== Grant privileges and roles needed for reading source maps + +++++ +Create a _source map_ user +++++ + +[[privileges-rum-source-mapping]] +==== APM Server RUM source mapping + +If <> is enabled, additional privileges are required to read source maps. + +To grant an APM Server user with the required privileges for reading RUM source maps from {es} directly without {kib}, +assign the user the following privileges: + +[options="header"] +|==== +|Type | Privilege | Purpose + +|Index +|`read` on `.apm-source-map` index +|Allow {beatname_uc} to read RUM source maps from {es} +|==== + +The above privileges should be sufficient for RUM source mapping to work properly +as long as {beatname_uc} communicates with {es} successfully. +If it fails, it may fallback to read source maps via {kib} if configured, +which requires additional {kib} privileges. +See {kibana-ref}/rum-sourcemap-api.html[RUM source map API] for more details. + //// *********************************** *********************************** *********************************** *********************************** From d2c362d8eaa57174500f5627606ac1237f47e340 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Oct 2023 18:57:07 +0200 Subject: [PATCH 04/16] feat: drop support for 32bit arch (#11871) * feat: drop support for 32bit arch * fix: remove additional i686 target Co-authored-by: Andrew Wilkins --------- Co-authored-by: Andrew Wilkins --- Makefile | 4 ---- internal/beatcmd/beat.go | 6 ------ packaging.mk | 17 +++-------------- x-pack/apm-server/aggregation.go | 2 -- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index ef501a19afd..a71f48b4045 100644 --- a/Makefile +++ b/Makefile @@ -37,9 +37,7 @@ CURRENT_DIR=$(shell dirname $(shell readlink -f $(firstword $(MAKEFILE_LIST)))) APM_SERVER_BINARIES:= \ build/apm-server-linux-amd64 \ - build/apm-server-linux-386 \ build/apm-server-linux-arm64 \ - build/apm-server-windows-386.exe \ build/apm-server-windows-amd64.exe \ build/apm-server-darwin-amd64 \ build/apm-server-darwin-arm64 @@ -64,7 +62,6 @@ $(APM_SERVER_BINARIES): build/apm-server-linux-%: GOOS=linux build/apm-server-darwin-%: GOOS=darwin build/apm-server-windows-%: GOOS=windows -build/apm-server-%-386 build/apm-server-%-386.exe: GOARCH=386 build/apm-server-%-amd64 build/apm-server-%-amd64.exe: GOARCH=amd64 build/apm-server-%-amd64 build/apm-server-%-amd64.exe: GOFLAGS+=-buildmode=pie build/apm-server-%-arm64 build/apm-server-%-arm64.exe: GOARCH=arm64 @@ -75,7 +72,6 @@ GOVERSIONINFO_FLAGS := \ -product-version "$(APM_SERVER_VERSION)" \ -comment "commit=$(GITCOMMIT)" -build/apm-server-windows-386.exe: x-pack/apm-server/versioninfo_windows_386.syso build/apm-server-windows-amd64.exe: x-pack/apm-server/versioninfo_windows_amd64.syso x-pack/apm-server/versioninfo_windows_amd64.syso: GOVERSIONINFO_FLAGS+=-64 x-pack/apm-server/versioninfo_%.syso: $(GOVERSIONINFO) $(GITREFFILE) packaging/versioninfo.json diff --git a/internal/beatcmd/beat.go b/internal/beatcmd/beat.go index 2ce87c017ec..a14dc566018 100644 --- a/internal/beatcmd/beat.go +++ b/internal/beatcmd/beat.go @@ -260,12 +260,6 @@ func (b *Beat) Run(ctx context.Context) error { defer logp.Info("%s stopped.", b.Info.Beat) logger := logp.NewLogger("") - if runtime.GOARCH == "386" { - logger.Warn("" + - "deprecation notice: support for 32-bit system target " + - "architecture will be removed in an upcoming version", - ) - } if runtime.GOOS == "darwin" { if host, err := sysinfo.Host(); err != nil { diff --git a/packaging.mk b/packaging.mk index 7d9c058088e..dffe5b7ecb5 100644 --- a/packaging.mk +++ b/packaging.mk @@ -106,25 +106,21 @@ WINDOWS_PACKAGE_FILES := \ # so we need to create arch-specific nfpm configuration files. build/nfpm-amd64.yml: PACKAGE_GOARCH=amd64 build/nfpm-arm64.yml: PACKAGE_GOARCH=arm64 -build/nfpm-386.yml: PACKAGE_GOARCH=386 build/nfpm-%.yml: packaging/nfpm.yml sed 's/$${GOARCH}/$(PACKAGE_GOARCH)/' $< | sed 's/$${APM_SERVER_VERSION}/${APM_SERVER_VERSION}/' > $@ -DEB_ARCH := i386 amd64 arm64 +DEB_ARCH := amd64 arm64 DEBS := $(patsubst %, $(DISTDIR)/apm-server-$(APM_SERVER_VERSION)-%.deb, $(DEB_ARCH)) DEBS += $(patsubst %, $(DISTDIR)/apm-server-$(APM_SERVER_VERSION)-SNAPSHOT-%.deb, $(DEB_ARCH)) -DEBS_386 := $(filter %-i386.deb, $(DEBS)) DEBS_AMD64 := $(filter %-amd64.deb, $(DEBS)) DEBS_ARM64 := $(filter %-arm64.deb, $(DEBS)) -RPM_ARCH := i686 x86_64 aarch64 +RPM_ARCH := x86_64 aarch64 RPMS := $(patsubst %, $(DISTDIR)/apm-server-$(APM_SERVER_VERSION)-%.rpm, $(RPM_ARCH)) RPMS += $(patsubst %, $(DISTDIR)/apm-server-$(APM_SERVER_VERSION)-SNAPSHOT-%.rpm, $(RPM_ARCH)) -RPMS_386 := $(filter %-i686.rpm, $(RPMS)) RPMS_AMD64 := $(filter %-x86_64.rpm, $(RPMS)) RPMS_ARM64 := $(filter %-aarch64.rpm, $(RPMS)) -$(DEBS_386) $(RPMS_386): $(NFPM) $(COMMON_PACKAGE_FILES) build/apm-server-linux-386 build/nfpm-386.yml $(DEBS_ARM64) $(RPMS_ARM64): $(NFPM) $(COMMON_PACKAGE_FILES) build/apm-server-linux-arm64 build/nfpm-arm64.yml $(DEBS_AMD64) $(RPMS_AMD64): $(NFPM) $(COMMON_PACKAGE_FILES) build/apm-server-linux-amd64 build/nfpm-amd64.yml @@ -136,19 +132,16 @@ $(DEBS_AMD64) $(RPMS_AMD64): $(NFPM) $(COMMON_PACKAGE_FILES) build/apm-server-li # # ARCHIVES_DIR holds intermediate directories created for building tgz and zip. ARCHIVES_DIR := build/archives -ARCHIVE_PLATFORMS := darwin-x86_64 linux-x86_64 linux-x86 linux-arm64 windows-x86_64 +ARCHIVE_PLATFORMS := darwin-x86_64 linux-x86_64 linux-arm64 windows-x86_64 ARCHIVE_PREFIX := $(ARCHIVES_DIR)/apm-server-$(APM_SERVER_VERSION) ARCHIVES := $(addprefix $(ARCHIVE_PREFIX)-, $(ARCHIVE_PLATFORMS)) ARCHIVES += $(addprefix $(ARCHIVE_PREFIX)-SNAPSHOT-, $(ARCHIVE_PLATFORMS)) $(ARCHIVE_PREFIX)-darwin-x86_64 $(ARCHIVE_PREFIX)-SNAPSHOT-darwin-x86_64: build/apm-server-darwin-amd64 $(COMMON_PACKAGE_FILES) $(ARCHIVE_PREFIX)-linux-x86_64 $(ARCHIVE_PREFIX)-SNAPSHOT-linux-x86_64: build/apm-server-linux-amd64 $(COMMON_PACKAGE_FILES) -$(ARCHIVE_PREFIX)-linux-x86 $(ARCHIVE_PREFIX)-SNAPSHOT-linux-x86: build/apm-server-linux-386 $(COMMON_PACKAGE_FILES) $(ARCHIVE_PREFIX)-linux-arm64 $(ARCHIVE_PREFIX)-SNAPSHOT-linux-arm64: build/apm-server-linux-arm64 $(COMMON_PACKAGE_FILES) $(ARCHIVE_PREFIX)-windows-x86_64 $(ARCHIVE_PREFIX)-SNAPSHOT-windows-x86_64: \ build/apm-server-windows-amd64.exe $(COMMON_PACKAGE_FILES) $(WINDOWS_PACKAGE_FILES) -$(ARCHIVE_PREFIX)-windows-x86 $(ARCHIVE_PREFIX)-SNAPSHOT-windows-x86: \ - build/apm-server-windows-386.exe $(COMMON_PACKAGE_FILES) $(WINDOWS_PACKAGE_FILES) $(ARCHIVE_PREFIX)-%: @rm -fr $@ && mkdir -p $@ @@ -182,15 +175,11 @@ $(DISTDIR)/%-docker-build-context.tar.gz: $(ARCHIVES_DIR)/%-docker-build-context PACKAGE_SUFFIXES := \ darwin-x86_64.tar.gz \ linux-x86_64.tar.gz \ - linux-x86.tar.gz \ linux-arm64.tar.gz \ windows-x86_64.zip \ - windows-x86.zip \ amd64.deb \ - i386.deb \ arm64.deb \ x86_64.rpm \ - i686.rpm \ aarch64.rpm build/dependencies-$(APM_SERVER_VERSION)-SNAPSHOT.csv: build/dependencies-$(APM_SERVER_VERSION).csv diff --git a/x-pack/apm-server/aggregation.go b/x-pack/apm-server/aggregation.go index 71c70e96359..856d4d7449b 100644 --- a/x-pack/apm-server/aggregation.go +++ b/x-pack/apm-server/aggregation.go @@ -2,8 +2,6 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -//go:build !386 - package main import ( From 791f582cba26bcf82cb8b6efbc15747049c77a22 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 20 Oct 2023 13:41:33 +0800 Subject: [PATCH 05/16] Fix service destination max group scaling based on memory (#11905) * Fix service destination max group scaling based on memory Fix regression introduced in #11739 - Fix bug where code is never executed - Fix wrong log message - Fix failing test --- internal/beater/beater.go | 2 +- internal/beater/config/aggregation.go | 12 ++---------- internal/beater/config/config_test.go | 2 +- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 2bbd53a1ae2..f6c7f5e4d53 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -253,7 +253,7 @@ func (s *Runner) Run(ctx context.Context) error { if s.config.Aggregation.ServiceDestinations.MaxGroups <= 0 { s.config.Aggregation.ServiceDestinations.MaxGroups = linearScaledValue(5_000, memLimitGB, 5_000) s.logger.Infof("Aggregation.ServiceDestinations.MaxGroups set to %d based on %0.1fgb of memory", - s.config.Aggregation.Transactions.MaxGroups, memLimitGB, + s.config.Aggregation.ServiceDestinations.MaxGroups, memLimitGB, ) } diff --git a/internal/beater/config/aggregation.go b/internal/beater/config/aggregation.go index 3163d30ba1e..8f4f9c76b57 100644 --- a/internal/beater/config/aggregation.go +++ b/internal/beater/config/aggregation.go @@ -17,10 +17,6 @@ package config -const ( - defaultServiceDestinationAggregationMaxGroups = 10000 -) - // AggregationConfig holds configuration related to various metrics aggregations. type AggregationConfig struct { MaxServices int `config:"max_services"` // if <= 0 then will be set based on memory limits @@ -36,7 +32,7 @@ type TransactionAggregationConfig struct { // ServiceDestinationAggregationConfig holds configuration related to span metrics aggregation for service maps. type ServiceDestinationAggregationConfig struct { - MaxGroups int `config:"max_groups" validate:"min=1"` + MaxGroups int `config:"max_groups"` // if <= 0 then will be set based on memory limits } // ServiceTransactionAggregationConfig holds configuration related to service transaction metrics aggregation. @@ -45,9 +41,5 @@ type ServiceTransactionAggregationConfig struct { } func defaultAggregationConfig() AggregationConfig { - return AggregationConfig{ - ServiceDestinations: ServiceDestinationAggregationConfig{ - MaxGroups: defaultServiceDestinationAggregationMaxGroups, - }, - } + return AggregationConfig{} } diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index b43f696dea3..71b4fd03a05 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -478,7 +478,7 @@ func TestUnpackConfig(t *testing.T) { MaxGroups: 0, // Default value is set as per memory limit }, ServiceDestinations: ServiceDestinationAggregationConfig{ - MaxGroups: 10000, + MaxGroups: 0, // Default value is set as per memory limit }, ServiceTransactions: ServiceTransactionAggregationConfig{ MaxGroups: 0, // Default value is set as per memory limit From 4308a86c235eeffc3c7bb204cf35202366152e72 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 20 Oct 2023 11:36:08 -0400 Subject: [PATCH 06/16] [updatecli] Bump elastic stack version to 8.12.0-129ce746 (#11908) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update k8s stack yaml files Made with ❤️️ by updatecli * chore: Update docker-compose.yml Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine --- docker-compose.yml | 6 +++--- testing/infra/k8s/base/stack/apm-server.yaml | 2 +- testing/infra/k8s/base/stack/elasticsearch.yaml | 2 +- testing/infra/k8s/base/stack/kibana.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 63024ccaa41..faf079f8cfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-ac05de1e-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-129ce746-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-ac05de1e-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-129ce746-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.12.0-ac05de1e-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.12.0-129ce746-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index 4e9c02e219e..840a3704944 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.12.0-ac05de1e-SNAPSHOT + version: 8.12.0-129ce746-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index 032b902e9ff..241e10d79c0 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.12.0-ac05de1e-SNAPSHOT + version: 8.12.0-129ce746-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index 07af48c96d1..ba883ca3ec0 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.12.0-ac05de1e-SNAPSHOT + version: 8.12.0-129ce746-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch From bc72990119758938c8598e7fb7dbab6f4efa6b30 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Oct 2023 02:13:43 +0200 Subject: [PATCH 07/16] changelog: add entry for datastream changes in 8.10 (#11914) --- changelogs/8.10.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/changelogs/8.10.asciidoc b/changelogs/8.10.asciidoc index 36b3381ab72..e7382633e71 100644 --- a/changelogs/8.10.asciidoc +++ b/changelogs/8.10.asciidoc @@ -46,6 +46,7 @@ Fix tail-based sampling discarding low throughput and low sample rate traces {pu ==== Breaking Changes - Aggregated metrics now consider global labels to be part of a service's identity, and high cardinality global labels may cause services to be obscured {pull}11386[11386] - Event protobuf encoding for tail-based sampling changed (to a more efficient encoding) for event timestamp and duration {pull}11386[11386] +- Use object notation for data_stream fields {pull}11311[11311] [float] ==== Bug fixes From 2d738c689c20b898a699105d693b6fc75793ceed Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 23 Oct 2023 12:20:35 -0400 Subject: [PATCH 08/16] [updatecli] Bump elastic stack version to 8.12.0-0f0dbad4 (#11921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update k8s stack yaml files Made with ❤️️ by updatecli * chore: Update docker-compose.yml Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine --- docker-compose.yml | 6 +++--- testing/infra/k8s/base/stack/apm-server.yaml | 2 +- testing/infra/k8s/base/stack/elasticsearch.yaml | 2 +- testing/infra/k8s/base/stack/kibana.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index faf079f8cfc..57f5906b95c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-129ce746-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-0f0dbad4-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-129ce746-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-0f0dbad4-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.12.0-129ce746-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.12.0-0f0dbad4-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index 840a3704944..aee70d413eb 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.12.0-129ce746-SNAPSHOT + version: 8.12.0-0f0dbad4-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index 241e10d79c0..4611d946c47 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.12.0-129ce746-SNAPSHOT + version: 8.12.0-0f0dbad4-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index ba883ca3ec0..0c7cbbd8271 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.12.0-129ce746-SNAPSHOT + version: 8.12.0-0f0dbad4-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch From 794cdd416c59c5ee5a9b6c3cfd9282a4fdcd2ebe Mon Sep 17 00:00:00 2001 From: Craig Rodrigues Date: Mon, 23 Oct 2023 17:48:30 -0700 Subject: [PATCH 09/16] docs: clarify the usage of the server information endpoint (#11881) Discussed in [case 01501264](https://support.elastic.co/cases/5008X00002UnA9UQAV) Signed-off-by: Craig Rodrigues --- docs/api-info.asciidoc | 46 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/api-info.asciidoc b/docs/api-info.asciidoc index 3bd7544ed54..4287e7da44a 100644 --- a/docs/api-info.asciidoc +++ b/docs/api-info.asciidoc @@ -8,22 +8,56 @@ This lightweight endpoint is useful as a server up/down health check. [[api-info-endpoint]] === Server Information endpoint -Send an `HTTP GET` request to the server information endpoint: +This is the server information endpoint: [source,bash] ------------------------------------------------------------ http(s)://{hostname}:{port}/ ------------------------------------------------------------ -This endpoint always returns an HTTP 200. +Sending an `HTTP GET` or `HTTP POST` request to the server information endpoint +will return an HTTP 200, indicating that the server is up. -If <> or a <> is configured, requests to this endpoint must be authenticated. +To configure authenticated access to the APM server, +the instructions at <> or <>, +must be followed to configure the correct permissions for APM access. + +If an <> or a <> is passed along with +the `HTTP GET` or `HTTP POST` request, in addition to an HTTP 200, +the response payload will include some information about the APM server. + +[float] +[[api-info-example-get-without-credentials]] +==== Example: GET, without credentials + +Example APM Server status request with GET, without credentials: + +["source","sh",subs="attributes"] +--------------------------------------------------------------------------- +curl --verbose -X GET http://127.0.0.1:8200 + +* Trying 127.0.0.1:8200... +* TCP_NODELAY set +* Connected to 127.0.0.1 (10.244.3.40) port 8200 (#0) +> GET / HTTP/1.1 +> Host: 127.0.0.1:8200 +> User-Agent: curl/7.68.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< X-Content-Type-Options: nosniff +< Date: Tue, 17 Oct 2023 22:04:05 GMT +< Content-Length: 0 +< +* Connection #0 to host 127.0.0.1 left intact +--------------------------------------------------------------------------- [float] -[[api-info-examples]] -==== Example +[[api-info-example-post-with-secret-token]] +==== Example: POST, with secret token -Example APM Server information request: +Example APM Server information request with POST, with a <>: ["source","sh",subs="attributes"] --------------------------------------------------------------------------- From 935754dccf984c42e0002e61e9a00cdfbe0df498 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Tue, 24 Oct 2023 17:39:13 +0800 Subject: [PATCH 10/16] Make OTLP input return partial success response (#11883) Bump apm-data, make OTLP input return partial success response --- NOTICE.txt | 36 +++++++++++++------------- changelogs/head.asciidoc | 1 + go.mod | 24 +++++++++--------- go.sum | 49 ++++++++++++++++++------------------ internal/beater/otlp/grpc.go | 27 +++++++++++++++----- internal/beater/otlp/http.go | 33 +++++++++++++++++++----- 6 files changed, 103 insertions(+), 67 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 06488552310..dbcb0ea5f06 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -362,11 +362,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v0.1.1-0.20230928144734-40b63726f3b7 +Version: v0.1.1-0.20231019084040-a0f7950e9775 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230928144734-40b63726f3b7/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20231019084040-a0f7950e9775/LICENSE: Apache License Version 2.0, January 2004 @@ -5282,11 +5282,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmotel/v -------------------------------------------------------------------------------- Dependency : go.elastic.co/apm/v2 -Version: v2.4.4 +Version: v2.4.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.4.4/LICENSE: +Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.4.5/LICENSE: Apache License Version 2.0, January 2004 @@ -5526,11 +5526,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/collector/consumer -Version: v0.81.0 +Version: v0.87.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/consumer@v0.81.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/consumer@v0.87.0/LICENSE: Apache License @@ -5738,11 +5738,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/cons -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/collector/pdata -Version: v1.0.0-rcv0013 +Version: v1.0.0-rcv0016 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.0.0-rcv0013/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/pdata@v1.0.0-rcv0016/LICENSE: Apache License @@ -6372,11 +6372,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/metric@v1 -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/otel/sdk/metric -Version: v0.40.0 +Version: v1.19.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/sdk/metric@v0.40.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/sdk/metric@v1.19.0/LICENSE: Apache License Version 2.0, January 2004 @@ -6611,11 +6611,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/zap -Version: v1.25.0 +Version: v1.26.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.25.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.26.0/LICENSE.txt: Copyright (c) 2016-2017 Uber Technologies, Inc. @@ -15082,11 +15082,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/common@v0.44 -------------------------------------------------------------------------------- Dependency : github.com/prometheus/procfs -Version: v0.10.1 +Version: v0.11.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.10.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.11.1/LICENSE: Apache License Version 2.0, January 2004 @@ -17439,11 +17439,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENS -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/collector/semconv -Version: v0.81.0 +Version: v0.87.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semconv@v0.81.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semconv@v0.87.0/LICENSE: Apache License @@ -17651,11 +17651,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/collector/semc -------------------------------------------------------------------------------- Dependency : go.opentelemetry.io/otel/sdk -Version: v1.17.0 +Version: v1.19.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/sdk@v1.17.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/sdk@v1.19.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 14ea9cdac2f..9652643289c 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -19,3 +19,4 @@ https://github.com/elastic/apm-server/compare/8.11\...main[View commits] [float] ==== Added - Update container base image to Ubuntu 22.04 {pull}11778[11778] +- Add support for returning partial success response in OTLP input {pull}11883[11883] diff --git a/go.mod b/go.mod index a879d3d7a62..ac3b20eb379 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 - github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7 + github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 github.com/elastic/elastic-agent-client/v7 v7.4.0 github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b @@ -23,7 +23,7 @@ require ( github.com/gofrs/flock v0.8.1 github.com/gofrs/uuid v4.4.0+incompatible github.com/gogo/protobuf v1.3.2 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru v0.5.4 @@ -42,15 +42,15 @@ require ( go.elastic.co/apm/module/apmgrpc/v2 v2.4.4 go.elastic.co/apm/module/apmhttp/v2 v2.4.4 go.elastic.co/apm/module/apmotel/v2 v2.4.4 - go.elastic.co/apm/v2 v2.4.4 + go.elastic.co/apm/v2 v2.4.5 go.elastic.co/fastjson v1.3.0 - go.opentelemetry.io/collector/consumer v0.81.0 - go.opentelemetry.io/collector/pdata v1.0.0-rcv0013 + go.opentelemetry.io/collector/consumer v0.87.0 + go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/metric v1.19.0 - go.opentelemetry.io/otel/sdk/metric v0.40.0 + go.opentelemetry.io/otel/sdk/metric v1.19.0 go.uber.org/automaxprocs v1.5.2 - go.uber.org/zap v1.25.0 + go.uber.org/zap v1.26.0 golang.org/x/net v0.17.0 golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 @@ -134,7 +134,7 @@ require ( github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect @@ -152,17 +152,17 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.elastic.co/apm/module/apmzap/v2 v2.4.4 // indirect go.elastic.co/ecszap v1.0.1 // indirect - go.opentelemetry.io/collector/semconv v0.81.0 // indirect - go.opentelemetry.io/otel/sdk v1.17.0 // indirect + go.opentelemetry.io/collector/semconv v0.87.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect - golang.org/x/mod v0.12.0 // indirect + golang.org/x/mod v0.13.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect diff --git a/go.sum b/go.sum index e3077e6c7ca..6b91478232e 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,6 @@ github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc h1:Keo7wQ7UODU github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -134,8 +132,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HEE0a5BDFmhEmJV/Uo4uzj/paHjd7yl6+KYguw= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= -github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7 h1:QcfCHAh1bHj9WN6y75bD6mBlIIlz9dJVKJSJVXCu4TM= -github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= +github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775 h1:IPo/osxNyRxn8akP082O51YyEtrPiUjkB7yxZV57AKQ= +github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 h1:Jkk/Vq79nJVDo4OtqFR/8TuzLqB9Zkv1sq2cyYceCm4= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93/go.mod h1:nVHoLIj1V0g6XdScm2ZbIezfuf3FKCHWTaFZ5DnPfSU= github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA= @@ -256,8 +254,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -438,8 +437,8 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -545,26 +544,26 @@ go.elastic.co/apm/module/apmotel/v2 v2.4.4 h1:DTIGTLiAZTjshCco2ljLnf35Q9m1MiWXBc go.elastic.co/apm/module/apmotel/v2 v2.4.4/go.mod h1:F14e5+BF+fmKs7jUH5i6C0tjzRUVrK4AbaNGw0z3mWk= go.elastic.co/apm/module/apmzap/v2 v2.4.4 h1:t1Ml4R5KapIoCJzem3d/bEaG0X7Gij0QijFPuf8wba8= go.elastic.co/apm/module/apmzap/v2 v2.4.4/go.mod h1:iYOVJKZEIWYr+VdD3IO0zjhC8j6my+v5vZNePNSgmIY= -go.elastic.co/apm/v2 v2.4.4 h1:KGbmUEtLNQ6sCFGpqTMhfRjqDOKP8IliO+efErKY5lU= -go.elastic.co/apm/v2 v2.4.4/go.mod h1:+CiBUdrrAGnGCL9TNx7tQz3BrfYV23L8Ljvotoc87so= +go.elastic.co/apm/v2 v2.4.5 h1:+01xPucS6YazBMYxbTSwNEsV9sgznFC0WW7x5dzxyPg= +go.elastic.co/apm/v2 v2.4.5/go.mod h1:+CiBUdrrAGnGCL9TNx7tQz3BrfYV23L8Ljvotoc87so= go.elastic.co/ecszap v1.0.1 h1:mBxqEJAEXBlpi5+scXdzL7LTFGogbuxipJC0KTZicyA= go.elastic.co/ecszap v1.0.1/go.mod h1:SVjazT+QgNeHSGOCUHvRgN+ZRj5FkB7IXQQsncdF57A= go.elastic.co/fastjson v1.3.0 h1:hJO3OsYIhiqiT4Fgu0ZxAECnKASbwgiS+LMW5oCopKs= go.elastic.co/fastjson v1.3.0/go.mod h1:K9vDh7O0ODsVKV2B5e2XYLY277QZaCbB3tS1SnARvko= -go.opentelemetry.io/collector/consumer v0.81.0 h1:8R2iCrSzD7T0RtC2Wh4GXxDiqla2vNhDokGW6Bcrfas= -go.opentelemetry.io/collector/consumer v0.81.0/go.mod h1:jS7+gAKdOx3lD3SnaBztBjUVpUYL3ee7fpoqI4p/gT8= -go.opentelemetry.io/collector/pdata v1.0.0-rcv0013 h1:4sONXE9hAX+4Di8m0bQ/KaoH3Mi+OPt04cXkZ7A8W3k= -go.opentelemetry.io/collector/pdata v1.0.0-rcv0013/go.mod h1:x09G/4KjEcDKNuWCjC5ZtnuDE0XEqiRwI+yrHSVjIy8= -go.opentelemetry.io/collector/semconv v0.81.0 h1:lCYNNo3powDvFIaTPP2jDKIrBiV1T92NK4QgL/aHYXw= -go.opentelemetry.io/collector/semconv v0.81.0/go.mod h1:TlYPtzvsXyHOgr5eATi43qEMqwSmIziivJB2uctKswo= +go.opentelemetry.io/collector/consumer v0.87.0 h1:oR5XKZoVF/hwz0FnrYPaHcbbQazHifMsxpENMR7ivvo= +go.opentelemetry.io/collector/consumer v0.87.0/go.mod h1:lui5rg1byAT7QPbCY733StCDc/TPxS3hVNXKoVQ3LsI= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 h1:qCPXSQCoD3qeWFb1RuIks8fw9Atxpk78bmtVdi15KhE= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0016/go.mod h1:OdN0alYOlYhHXu6BDlGehrZWgtBuiDsz/rlNeJeXiNg= +go.opentelemetry.io/collector/semconv v0.87.0 h1:BsG1jdLLRCBRlvUujk4QA86af7r/ZXnizczQpEs/gg8= +go.opentelemetry.io/collector/semconv v0.87.0/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE= -go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= -go.opentelemetry.io/otel/sdk/metric v0.40.0 h1:qOM29YaGcxipWjL5FzpyZDpCYrDREvX0mVlmXdOjCHU= -go.opentelemetry.io/otel/sdk/metric v0.40.0/go.mod h1:dWxHtdzdJvg+ciJUKLTKwrMe5P6Dv3FyDbh8UkfgkVs= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= +go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -579,8 +578,8 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -602,8 +601,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -695,8 +694,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/beater/otlp/grpc.go b/internal/beater/otlp/grpc.go index 5c97ce7d55b..82722d1c92d 100644 --- a/internal/beater/otlp/grpc.go +++ b/internal/beater/otlp/grpc.go @@ -95,8 +95,13 @@ func (s *tracesService) Export(ctx context.Context, req ptraceotlp.ExportRequest if td.SpanCount() == 0 { return ptraceotlp.NewExportResponse(), nil } - err := s.consumer.ConsumeTraces(ctx, td) - return ptraceotlp.NewExportResponse(), err + resp := ptraceotlp.NewExportResponse() + result, err := s.consumer.ConsumeTracesWithResult(ctx, td) + if err == nil && result.RejectedSpans > 0 { + resp.PartialSuccess().SetRejectedSpans(result.RejectedSpans) + resp.PartialSuccess().SetErrorMessage(result.ErrorMessage) + } + return resp, err } type metricsService struct { @@ -109,8 +114,13 @@ func (s *metricsService) Export(ctx context.Context, req pmetricotlp.ExportReque if md.DataPointCount() == 0 { return pmetricotlp.NewExportResponse(), nil } - err := s.consumer.ConsumeMetrics(ctx, md) - return pmetricotlp.NewExportResponse(), err + resp := pmetricotlp.NewExportResponse() + result, err := s.consumer.ConsumeMetricsWithResult(ctx, md) + if err == nil && result.RejectedDataPoints > 0 { + resp.PartialSuccess().SetRejectedDataPoints(result.RejectedDataPoints) + resp.PartialSuccess().SetErrorMessage(result.ErrorMessage) + } + return resp, err } type logsService struct { @@ -123,6 +133,11 @@ func (s *logsService) Export(ctx context.Context, req plogotlp.ExportRequest) (p if ld.LogRecordCount() == 0 { return plogotlp.NewExportResponse(), nil } - err := s.consumer.ConsumeLogs(ctx, ld) - return plogotlp.NewExportResponse(), err + resp := plogotlp.NewExportResponse() + result, err := s.consumer.ConsumeLogsWithResult(ctx, ld) + if err == nil && result.RejectedLogRecords > 0 { + resp.PartialSuccess().SetRejectedLogRecords(result.RejectedLogRecords) + resp.PartialSuccess().SetErrorMessage(result.ErrorMessage) + } + return resp, err } diff --git a/internal/beater/otlp/http.go b/internal/beater/otlp/http.go index ecba68a5248..aaf9a075066 100644 --- a/internal/beater/otlp/http.go +++ b/internal/beater/otlp/http.go @@ -78,11 +78,18 @@ func (h HTTPHandlers) HandleTraces(w http.ResponseWriter, r *http.Request) { h.writeError(w, err, http.StatusBadRequest) return } - if err := h.consumer.ConsumeTraces(r.Context(), req.Traces()); err != nil { + var result otlp.ConsumeTracesResult + var err error + if result, err = h.consumer.ConsumeTracesWithResult(r.Context(), req.Traces()); err != nil { h.writeError(w, err, http.StatusInternalServerError) return } - if err := h.writeResponse(w, ptraceotlp.NewExportResponse()); err != nil { + resp := ptraceotlp.NewExportResponse() + if result.RejectedSpans > 0 { + resp.PartialSuccess().SetRejectedSpans(result.RejectedSpans) + resp.PartialSuccess().SetErrorMessage(result.ErrorMessage) + } + if err := h.writeResponse(w, resp); err != nil { h.writeError(w, err, http.StatusInternalServerError) return } @@ -96,11 +103,18 @@ func (h HTTPHandlers) HandleMetrics(w http.ResponseWriter, r *http.Request) { h.writeError(w, err, http.StatusBadRequest) return } - if err := h.consumer.ConsumeMetrics(r.Context(), req.Metrics()); err != nil { + var result otlp.ConsumeMetricsResult + var err error + if result, err = h.consumer.ConsumeMetricsWithResult(r.Context(), req.Metrics()); err != nil { h.writeError(w, err, http.StatusInternalServerError) return } - if err := h.writeResponse(w, pmetricotlp.NewExportResponse()); err != nil { + resp := pmetricotlp.NewExportResponse() + if result.RejectedDataPoints > 0 { + resp.PartialSuccess().SetRejectedDataPoints(result.RejectedDataPoints) + resp.PartialSuccess().SetErrorMessage(result.ErrorMessage) + } + if err := h.writeResponse(w, resp); err != nil { h.writeError(w, err, http.StatusInternalServerError) return } @@ -114,11 +128,18 @@ func (h HTTPHandlers) HandleLogs(w http.ResponseWriter, r *http.Request) { h.writeError(w, err, http.StatusBadRequest) return } - if err := h.consumer.ConsumeLogs(r.Context(), req.Logs()); err != nil { + var result otlp.ConsumeLogsResult + var err error + if result, err = h.consumer.ConsumeLogsWithResult(r.Context(), req.Logs()); err != nil { h.writeError(w, err, http.StatusInternalServerError) return } - if err := h.writeResponse(w, plogotlp.NewExportResponse()); err != nil { + resp := plogotlp.NewExportResponse() + if result.RejectedLogRecords > 0 { + resp.PartialSuccess().SetRejectedLogRecords(result.RejectedLogRecords) + resp.PartialSuccess().SetErrorMessage(result.ErrorMessage) + } + if err := h.writeResponse(w, resp); err != nil { h.writeError(w, err, http.StatusInternalServerError) return } From 5584a8aa2a999a3da3f656de66a15ace58f03665 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 24 Oct 2023 13:40:41 +0200 Subject: [PATCH 11/16] fix: correct base Docker image names (#11926) --- packaging/ironbank/Dockerfile | 2 +- packaging/ironbank/hardening_manifest.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/ironbank/Dockerfile b/packaging/ironbank/Dockerfile index 147733484c6..16fd92374ba 100644 --- a/packaging/ironbank/Dockerfile +++ b/packaging/ironbank/Dockerfile @@ -3,7 +3,7 @@ # Extract APM Server and make various file manipulations. ################################################################################ ARG BASE_REGISTRY=registry1.dsop.io -ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 +ARG BASE_IMAGE=redhat/ubi/ubi9 ARG BASE_TAG=9.2 FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS prep_files diff --git a/packaging/ironbank/hardening_manifest.yaml b/packaging/ironbank/hardening_manifest.yaml index 2b8ad57f2bc..427862d8a81 100644 --- a/packaging/ironbank/hardening_manifest.yaml +++ b/packaging/ironbank/hardening_manifest.yaml @@ -13,7 +13,7 @@ tags: # Build args passed to Dockerfile ARGs args: - BASE_IMAGE: "ironbank/redhat/ubi/ubi9" + BASE_IMAGE: "redhat/ubi/ubi9" BASE_TAG: "9.2" ELASTIC_STACK: "${APM_SERVER_VERSION}" ELASTIC_PRODUCT: "apm-server" From cde4794f17b2ae2b7fdad2737b994aed2d8c1bc0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 24 Oct 2023 16:41:35 +0200 Subject: [PATCH 12/16] ubi9: update the reference as done with the ironbank (#11927) --- packaging.mk | 8 ++++---- tools/notice/dependencies.csv.tmpl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging.mk b/packaging.mk index dffe5b7ecb5..36afe0b0f0d 100644 --- a/packaging.mk +++ b/packaging.mk @@ -28,13 +28,13 @@ DOCKER_BUILD_ARGS := \ DOCKER_IMAGES := \ build/docker/apm-server-$(APM_SERVER_VERSION).txt \ build/docker/apm-server-$(APM_SERVER_VERSION)-SNAPSHOT.txt \ - build/docker/apm-server-ubi8-$(APM_SERVER_VERSION).txt \ - build/docker/apm-server-ubi8-$(APM_SERVER_VERSION)-SNAPSHOT.txt + build/docker/apm-server-ubi9-$(APM_SERVER_VERSION).txt \ + build/docker/apm-server-ubi9-$(APM_SERVER_VERSION)-SNAPSHOT.txt build/docker/%.txt: DOCKER_IMAGE_TAG := docker.elastic.co/apm/apm-server:% build/docker/%.txt: VERSION := $(APM_SERVER_VERSION) build/docker/%-SNAPSHOT.txt: VERSION := $(APM_SERVER_VERSION)-SNAPSHOT -build/docker/apm-server-ubi8-%.txt: DOCKER_BUILD_ARGS+=--build-arg BASE_IMAGE=docker.elastic.co/ubi8/ubi-minimal +build/docker/apm-server-ubi9-%.txt: DOCKER_BUILD_ARGS+=--build-arg BASE_IMAGE=docker.elastic.co/ubi9/ubi-minimal .PHONY: $(DOCKER_IMAGES) $(DOCKER_IMAGES): @@ -43,7 +43,7 @@ $(DOCKER_IMAGES): # Docker image tarballs. We distribute UBI8 Docker images only for AMD64. DOCKER_IMAGE_SUFFIX := docker-image$(if $(findstring arm64,$(GOARCH)),-arm64).tar.gz -DOCKER_IMAGE_PREFIXES := apm-server $(if $(findstring amd64,$(GOARCH)), apm-server-ubi8) +DOCKER_IMAGE_PREFIXES := apm-server $(if $(findstring amd64,$(GOARCH)), apm-server-ubi9) DOCKER_IMAGE_RELEASE_TARBALLS := $(patsubst %, $(DISTDIR)/%-$(APM_SERVER_VERSION)-$(DOCKER_IMAGE_SUFFIX), $(DOCKER_IMAGE_PREFIXES)) DOCKER_IMAGE_SNAPSHOT_TARBALLS := $(patsubst %, $(DISTDIR)/%-$(APM_SERVER_VERSION)-SNAPSHOT-$(DOCKER_IMAGE_SUFFIX), $(DOCKER_IMAGE_PREFIXES)) diff --git a/tools/notice/dependencies.csv.tmpl b/tools/notice/dependencies.csv.tmpl index b7e346054a4..9595bc11d19 100644 --- a/tools/notice/dependencies.csv.tmpl +++ b/tools/notice/dependencies.csv.tmpl @@ -5,4 +5,4 @@ {{- end -}} name,url,version,revision,license,sourceURL{{ template "depInfo" .Direct }}{{ template "depInfo" .Indirect }} -Red Hat Universal Base Image minimal,https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8,8,,Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf,https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/8/ubi-minimal-8-source.tar.gz +Red Hat Universal Base Image minimal,https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5,9,,Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf,https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/9/ubi-minimal-9-source.tar.gz From e7e4853cfe20d96e64420bd7ea2b266e2b2d451a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 25 Oct 2023 09:46:30 +0200 Subject: [PATCH 13/16] mergify: add backport label for the 7.17 backports (#11931) This will help with the auto-merge --- .mergify.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 5cd9cbc8d86..1f3ae86e04f 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -92,6 +92,8 @@ pull_request_rules: - "{{ author }}" branches: - "7.17" + labels: + - "backport" title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" - name: backport patches to 7.16 branch conditions: From 3b737c9199dee8f59623cde3124548286b88a843 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 25 Oct 2023 12:28:09 -0400 Subject: [PATCH 14/16] [updatecli] Update to elastic/beats@bbf0111c1d50 (#11912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update to elastic/beats@42f2f9426043 Made with ❤️️ by updatecli * chore: Update to elastic/beats@2b7dbeaa53d5 Made with ❤️️ by updatecli * build: fix compile erros due to beats recent changes * chore: Update to elastic/beats@effe78ceac5b Made with ❤️️ by updatecli * chore: Update to elastic/beats@bbf0111c1d50 Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine Co-authored-by: kruskal <99559985+kruskall@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- NOTICE.txt | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- internal/beater/beatertest/output.go | 2 +- internal/elasticsearch/config_test.go | 1 + 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index dbcb0ea5f06..c7e934b2a8c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 +Version: v7.0.0-alpha2.0.20231025101650-bbf0111c1d50 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231018064857-d3c42ef75b93/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20231025101650-bbf0111c1d50/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -829,11 +829,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.3.15-0.20230913212237-dbdaf18c898b +Version: v0.6.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.15-0.20230913212237-dbdaf18c898b/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.6.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index ac3b20eb379..5e42690905c 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,9 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231025101650-bbf0111c1d50 github.com/elastic/elastic-agent-client/v7 v7.4.0 - github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b + github.com/elastic/elastic-agent-libs v0.6.0 github.com/elastic/elastic-agent-system-metrics v0.7.0 github.com/elastic/gmux v0.3.1 github.com/elastic/go-docappender v0.2.1-0.20230829163624-c69a1cf8ce35 diff --git a/go.sum b/go.sum index 6b91478232e..2929867443e 100644 --- a/go.sum +++ b/go.sum @@ -134,14 +134,14 @@ github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HE github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775 h1:IPo/osxNyRxn8akP082O51YyEtrPiUjkB7yxZV57AKQ= github.com/elastic/apm-data v0.1.1-0.20231019084040-a0f7950e9775/go.mod h1:z4iJVl8vyQa5v5o7UapWGHTsycBKsKfJfILuf2TZpYo= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93 h1:Jkk/Vq79nJVDo4OtqFR/8TuzLqB9Zkv1sq2cyYceCm4= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231018064857-d3c42ef75b93/go.mod h1:nVHoLIj1V0g6XdScm2ZbIezfuf3FKCHWTaFZ5DnPfSU= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231025101650-bbf0111c1d50 h1:JPN/Jnn8veEgNjnmfy7p88NOGZ6GVOwipURCaqgvHFY= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20231025101650-bbf0111c1d50/go.mod h1:nosWvRbRXMSEvMjpXyUw03XlEc7uejUZ3GSX2KSl6uM= github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA= github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= -github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM= -github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ= +github.com/elastic/elastic-agent-libs v0.6.0 h1:HnL/OpAzIHlK8y1J69XQuAx4tlCzd6e2kldMHvXARvY= +github.com/elastic/elastic-agent-libs v0.6.0/go.mod h1:K6U+n84siZ66ZyG36h1/x+fw1oIZbFXEypAC6KSiFOg= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ= diff --git a/internal/beater/beatertest/output.go b/internal/beater/beatertest/output.go index 64fe7a6a230..a72dcf249a9 100644 --- a/internal/beater/beatertest/output.go +++ b/internal/beater/beatertest/output.go @@ -90,7 +90,7 @@ func makeNullOutput( _ *agentconfig.C, ) (outputs.Group, error) { client := nullOutput{observer: observer} - return outputs.Success(1, 0, client) + return outputs.Success(agentconfig.Namespace{}, 1, 0, client) } func (nullOutput) String() string { diff --git a/internal/elasticsearch/config_test.go b/internal/elasticsearch/config_test.go index 21c17cdef73..2676d31a6b9 100644 --- a/internal/elasticsearch/config_test.go +++ b/internal/elasticsearch/config_test.go @@ -170,6 +170,7 @@ func TestBeatsConfigSynced(t *testing.T) { knownUnhandled := []string{ "backoff", // we have our own version with its own type + "queue", "bulk_max_size", "escape_html", // TODO Kerberos auth (https://github.com/elastic/apm-server/issues/3794) From 3c4dea9f6e943f737ffec24e7c717aec55092117 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 06:01:28 +0200 Subject: [PATCH 15/16] build(deps): bump google.golang.org/grpc in /systemtest (#11944) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.57.0 to 1.57.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.57.0...v1.57.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- systemtest/go.mod | 2 +- systemtest/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/systemtest/go.mod b/systemtest/go.mod index 54984d3bc93..dd8be350785 100644 --- a/systemtest/go.mod +++ b/systemtest/go.mod @@ -36,7 +36,7 @@ require ( golang.org/x/sync v0.3.0 golang.org/x/sys v0.13.0 golang.org/x/time v0.3.0 - google.golang.org/grpc v1.57.0 + google.golang.org/grpc v1.57.1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/systemtest/go.sum b/systemtest/go.sum index a59f2ce8fc6..be6dae516b3 100644 --- a/systemtest/go.sum +++ b/systemtest/go.sum @@ -292,8 +292,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1: google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= +google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= From 587199425e7cc53747747b7887964c120bdb3b3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 00:17:13 -0400 Subject: [PATCH 16/16] build(deps): bump google.golang.org/grpc from 1.56.2 to 1.56.3 in /tools (#11945) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.56.2 to 1.56.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.56.2...v1.56.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com> --- tools/go.mod | 2 +- tools/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/go.mod b/tools/go.mod index d944adf45b3..4420589b94f 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -245,7 +245,7 @@ require ( google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.56.2 // indirect + google.golang.org/grpc v1.56.3 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index 443131bbece..263ea84e67b 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -1251,8 +1251,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=