diff --git a/.mergify.yml b/.mergify.yml index d06ba9c242c..5cd9cbc8d86 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -288,3 +288,17 @@ pull_request_rules: labels: - "backport" title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + - name: backport patches to 8.11 branch + conditions: + - merged + - base=main + - label=backport-8.11 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.11" + labels: + - "backport" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" diff --git a/NOTICE.txt b/NOTICE.txt index 2f424e42ac3..386d5489346 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -362,11 +362,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v0.1.1-0.20230908092227-4426f61e0c5c +Version: v0.1.1-0.20230928144734-40b63726f3b7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230908092227-4426f61e0c5c/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230928144734-40b63726f3b7/LICENSE: Apache License Version 2.0, January 2004 @@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20230920140255-24c3388ed6d8 +Version: v7.0.0-alpha2.0.20230928231718-49cc14d35d5e Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20230920140255-24c3388ed6d8/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20230928231718-49cc14d35d5e/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -596,11 +596,11 @@ License Version 2.0. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.3.0 +Version: v7.4.0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.3.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.4.0/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -829,11 +829,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.3.13 +Version: v0.3.15-0.20230913212237-dbdaf18c898b Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.13/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.15-0.20230913212237-dbdaf18c898b/LICENSE: Apache License Version 2.0, January 2004 @@ -1673,11 +1673,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-docappender@ -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-elasticsearch/v8 -Version: v8.9.0 +Version: v8.10.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.10.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml index 521d2986547..6906aa2b13d 100644 --- a/apmpackage/apm/changelog.yml +++ b/apmpackage/apm/changelog.yml @@ -1,5 +1,13 @@ - version: generated changes: + - description: Placeholder + type: enhancement + link: https://github.com/elastic/apm-server/pull/123 +- version: 8.11.0 + changes: + - description: Add geoip processing to add client.geo.* fields for app_logs + type: enhancement + link: https://github.com/elastic/apm-server/pull/11699 - description: Define data retentions to support DLM type: enhancement link: https://github.com/elastic/apm-server/pull/11539 diff --git a/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml b/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml index ee41159d1d0..0d8331bd71f 100644 --- a/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml +++ b/apmpackage/apm/data_stream/app_logs/elasticsearch/ingest_pipeline/default.yml @@ -9,3 +9,5 @@ processors: name: observer_ids - pipeline: name: remove_ecs_version + - pipeline: + name: client_geoip diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 4776dce2fb2..a7d8d53ba2e 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -15,13 +15,14 @@ https://github.com/elastic/apm-server/compare/8.10\...main[View commits] - Add back gzip support for grpc otlp endpoint {pull}11434[11434] - Correctly mark jvm.memory.non_heap.pool.* and jvm.fd.* metrics as internal {pull}11303[11303] - Fix tail-based sampling discarding low throughput and low sample rate traces {pull}11642[11642] +- Add memory based autoscaling for service destination aggregation groups {pull}11739[11739] [float] ==== Intake API Changes [float] ==== Added -- Support and define DLM data retention period in the apmpackage +- Support and define DLM data retention period in the apmpackage {pull}11539[11539] - Expose new metrics into the local batch processor {pull}11582[11582] : - http.server.request.count - http.server.request.duration @@ -35,3 +36,5 @@ https://github.com/elastic/apm-server/compare/8.10\...main[View commits] - grpc.server.response.errors.count - grpc.server.errors.timeout - grpc.server.errors.ratelimit +- Add geoip processing to app_logs ingest pipeline on `client.ip` {pull}11699[11699] + diff --git a/cmd/intake-receiver/version.go b/cmd/intake-receiver/version.go index a13c8b643ee..dbe4d25625d 100644 --- a/cmd/intake-receiver/version.go +++ b/cmd/intake-receiver/version.go @@ -18,4 +18,4 @@ package main // version matches the APM Server's version -const version = "8.11.0" +const version = "8.12.0" diff --git a/docker-compose.yml b/docker-compose.yml index 283f82580ec..db7966546a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-abc4d077-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-4d413233-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.11.0-abc4d077-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.11.0-4d413233-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.11.0-abc4d077-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.11.0-4d413233-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/docs/configure/agent-config.asciidoc b/docs/configure/agent-config.asciidoc index 566aaa6a4a8..6261a389827 100644 --- a/docs/configure/agent-config.asciidoc +++ b/docs/configure/agent-config.asciidoc @@ -71,7 +71,4 @@ rejecting fetch request: no valid elasticsearch config This occurs because the user or API key set in either `apm-server.agent.config.elasticsearch` or `output.elasticsearch` (if `apm-server.agent.config.elasticsearch` is not set) does not have adequate permissions to read source maps from {es}. -To fix this error, add the following index-level privileges to the API key: - -* `read` privileges on the `.apm-agent-configuration` index -* `allow_restricted_indices: true` +To fix this error, ensure that {beatname_uc} has all the required privileges. See <> for more details. diff --git a/docs/data-model.asciidoc b/docs/data-model.asciidoc index 918c48f2c23..5e6f3ad9724 100644 --- a/docs/data-model.asciidoc +++ b/docs/data-model.asciidoc @@ -548,9 +548,10 @@ which is 500 transaction groups per service per GB of APM Server. ** For service-transaction metrics, there is an additional limit of 1000 total service transaction groups per GB of APM Server, and each service may only consume up to 10% of the service transaction groups, which is 100 service transaction groups per service per GB of APM Server. -** For service-destination metrics, there is an additional limit of a constant 10000 total service destination groups, +** For service-destination metrics, there is an additional limit of 5000 total service destination groups per GB of APM Server +starting with 10000 service destination groups for 1 GB APM Server, and each service may only consume up to 10% of the service destination groups, -which is 1000 service destination groups per service. +which is 1000 service destination groups for 1GB APM Server with 500 increment per GB of APM Server. ** For service-summary metrics, there is no additional limit. In the above, a service is defined as a combination of `service.name`, `service.environment`, `service.language.name` and `agent.name`. diff --git a/docs/feature-roles.asciidoc b/docs/feature-roles.asciidoc index 8b60c1abdb0..65a801ce8b2 100644 --- a/docs/feature-roles.asciidoc +++ b/docs/feature-roles.asciidoc @@ -319,25 +319,39 @@ PUT _security/role/apm_api_key <1> ++++ [[privileges-agent-central-config-server]] -==== APM Server central configuration management +==== APM Server agent central configuration management APM Server acts as a proxy between your APM agents and the {apm-app}. The {apm-app} communicates any changed settings to APM Server so that your agents only need to poll the Server to determine which central configuration settings have changed. -To grant an APM Server user with the required privileges for managing central configuration, +To grant an APM Server user with the required privileges for managing central configuration in {es} without {kib}, assign the user the following privileges: [options="header"] |==== |Type | Privilege | Purpose +| Index +|`read` on `.apm-agent-configuration` index +|Allow {beatname_uc} to manage central configurations in {es} +|==== + +The above privileges should be sufficient for APM agent central configuration to work properly +as long as {beatname_uc} communicates with {es} successfully. +If it fails, it may fallback to read agent central configuration via {kib} if configured, +which requires the following privileges: + +[options="header"] +|==== +|Type | Privilege | Purpose + | Spaces |`Read` on {beat_kib_app} |Allow {beatname_uc} to manage central configurations via the {beat_kib_app} |==== -TIP: Looking for privileges and roles needed use central configuration from the {apm-app} or {apm-app} API? +TIP: Looking for privileges and roles needed to use central configuration from the {apm-app} or {apm-app} API? See {kibana-ref}/apm-app-central-config-user.html[{apm-app} central configuration user]. //// diff --git a/go.mod b/go.mod index 391a3557d77..ff983da4989 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 - github.com/elastic/apm-data v0.1.1-0.20230908092227-4426f61e0c5c - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230920140255-24c3388ed6d8 - github.com/elastic/elastic-agent-client/v7 v7.3.0 - github.com/elastic/elastic-agent-libs v0.3.13 + github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230928231718-49cc14d35d5e + github.com/elastic/elastic-agent-client/v7 v7.4.0 + github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b github.com/elastic/elastic-agent-system-metrics v0.6.1 github.com/elastic/gmux v0.2.0 github.com/elastic/go-docappender v0.2.1-0.20230829163624-c69a1cf8ce35 - github.com/elastic/go-elasticsearch/v8 v8.9.0 + github.com/elastic/go-elasticsearch/v8 v8.10.0 github.com/elastic/go-sysinfo v1.11.1 github.com/elastic/go-ucfg v0.8.6 github.com/go-sourcemap/sourcemap v2.1.3+incompatible diff --git a/go.sum b/go.sum index 0bc576f77b3..6eb3f53d330 100644 --- a/go.sum +++ b/go.sum @@ -173,16 +173,16 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HEE0a5BDFmhEmJV/Uo4uzj/paHjd7yl6+KYguw= github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo= -github.com/elastic/apm-data v0.1.1-0.20230908092227-4426f61e0c5c h1:THld+d7v6wEwM1v+gzI9j0kktOI/tZHRWJVIVUGHctc= -github.com/elastic/apm-data v0.1.1-0.20230908092227-4426f61e0c5c/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230920140255-24c3388ed6d8 h1:B09aM9DyjovwqXH7f2wIBXIWWJ9CR74T8+L+HCbzpik= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230920140255-24c3388ed6d8/go.mod h1:6VoNnMqI4R6FUggDK0bGPNnL3zdk4/rW5CUSLZBqnls= +github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7 h1:QcfCHAh1bHj9WN6y75bD6mBlIIlz9dJVKJSJVXCu4TM= +github.com/elastic/apm-data v0.1.1-0.20230928144734-40b63726f3b7/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230928231718-49cc14d35d5e h1:BPKkwmQMVP3py9S12UIhmdnHjpcYDfMo2NafF6E/B/A= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230928231718-49cc14d35d5e/go.mod h1:qgXAHO18gZmQhE8bQwmigeANroMnUq+98hgbCHh+hjk= github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+PiWyZ4I4F4LqrCYgYlNc= github.com/elastic/elastic-agent-autodiscover v0.6.2/go.mod h1:yXYKFAG+Py+TcE4CCR8EAbJiYb+6Dz9sCDoWgOveqtU= -github.com/elastic/elastic-agent-client/v7 v7.3.0 h1:LugKtBXK7bp4SFL/uQqGU/f4Ppx12Jk5a36voGabLa0= -github.com/elastic/elastic-agent-client/v7 v7.3.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= -github.com/elastic/elastic-agent-libs v0.3.13 h1:qFiBWeBfjsBId+i31rggyW2ZjzA9qBRz7wIiy+rkcvc= -github.com/elastic/elastic-agent-libs v0.3.13/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ= +github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= +github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= +github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM= +github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.6.1 h1:LCN1lvQTkdUuU/rKlpKyVMDU/G/I8/iZWCaW6K+mo4o= @@ -194,8 +194,8 @@ github.com/elastic/gmux v0.2.0 h1:HzaJ6FQAZzKJ2RTrINIfDXN1voO5EEEJKLb1Hlrn8pw= github.com/elastic/gmux v0.2.0/go.mod h1:6+9rYPXZXAyCIb7g3WQ0OVWoLNpU/xHz2VXUrtw6BUg= github.com/elastic/go-docappender v0.2.1-0.20230829163624-c69a1cf8ce35 h1:zBB2RqyEcPlJ16psSXIV6qhhbFB+Bf9RZrgcCk2vCiY= github.com/elastic/go-docappender v0.2.1-0.20230829163624-c69a1cf8ce35/go.mod h1:QddPogzCYGtzbR8ZFGRG2kvQ+vRG4A4JGMoH82G1dT8= -github.com/elastic/go-elasticsearch/v8 v8.9.0 h1:8xtmYjUkqtahl50E0Bg/wjKI7K63krJrrLipbNj/fCU= -github.com/elastic/go-elasticsearch/v8 v8.9.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE= +github.com/elastic/go-elasticsearch/v8 v8.10.0 h1:ALg3DMxSrx07YmeMNcfPf7cFh1Ep2+Qa19EOXTbwr2k= +github.com/elastic/go-elasticsearch/v8 v8.10.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE= github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 0361ba19731..2bbd53a1ae2 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -230,23 +230,30 @@ func (s *Runner) Run(ctx context.Context) error { } if s.config.Aggregation.MaxServices <= 0 { - s.config.Aggregation.MaxServices = maxGroupsForAggregation(memLimitGB) + s.config.Aggregation.MaxServices = linearScaledValue(1_000, memLimitGB, 0) s.logger.Infof("Aggregation.MaxServices set to %d based on %0.1fgb of memory", s.config.Aggregation.MaxServices, memLimitGB, ) } + if s.config.Aggregation.ServiceTransactions.MaxGroups <= 0 { + s.config.Aggregation.ServiceTransactions.MaxGroups = linearScaledValue(1_000, memLimitGB, 0) + s.logger.Infof("Aggregation.ServiceTransactions.MaxGroups for service aggregation set to %d based on %0.1fgb of memory", + s.config.Aggregation.ServiceTransactions.MaxGroups, memLimitGB, + ) + } + if s.config.Aggregation.Transactions.MaxGroups <= 0 { - s.config.Aggregation.Transactions.MaxGroups = maxTxGroupsForAggregation(memLimitGB) + s.config.Aggregation.Transactions.MaxGroups = linearScaledValue(5_000, memLimitGB, 0) s.logger.Infof("Aggregation.Transactions.MaxGroups set to %d based on %0.1fgb of memory", s.config.Aggregation.Transactions.MaxGroups, memLimitGB, ) } - if s.config.Aggregation.ServiceTransactions.MaxGroups <= 0 { - s.config.Aggregation.ServiceTransactions.MaxGroups = maxGroupsForAggregation(memLimitGB) - s.logger.Infof("Aggregation.ServiceTransactions.MaxGroups for service aggregation set to %d based on %0.1fgb of memory", - s.config.Aggregation.ServiceTransactions.MaxGroups, memLimitGB, + if s.config.Aggregation.ServiceDestinations.MaxGroups <= 0 { + s.config.Aggregation.ServiceDestinations.MaxGroups = linearScaledValue(5_000, memLimitGB, 5_000) + s.logger.Infof("Aggregation.ServiceDestinations.MaxGroups set to %d based on %0.1fgb of memory", + s.config.Aggregation.Transactions.MaxGroups, memLimitGB, ) } @@ -568,26 +575,14 @@ func maxConcurrentDecoders(memLimitGB float64) uint { return decoders } -// maxGroupsForAggregation calculates the maximum service groups that a -// particular memory limit can have. This will be scaled linearly for bigger -// instances. -func maxGroupsForAggregation(memLimitGB float64) int { - const maxMemGB = 64 - if memLimitGB > maxMemGB { - memLimitGB = maxMemGB - } - return int(memLimitGB * 1_000) -} - -// maxTxGroupsForAggregation calculates the maximum transaction groups that a -// particular memory limit can have. This will be scaled linearly for bigger -// instances. -func maxTxGroupsForAggregation(memLimitGB float64) int { +// linearScaledValue calculates linearly scaled value based on memory limit using +// the formula y = (perGBIncrement * memLimitGB) + constant +func linearScaledValue(perGBIncrement, memLimitGB, constant float64) int { const maxMemGB = 64 if memLimitGB > maxMemGB { memLimitGB = maxMemGB } - return int(memLimitGB * 5_000) + return int(memLimitGB*perGBIncrement + constant) } // waitReady waits until the server is ready to index events. diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index d564fed1816..b43f696dea3 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -53,7 +53,10 @@ func TestUnpackConfig(t *testing.T) { kibanaHeadersConfig := DefaultConfig() kibanaHeadersConfig.Kibana.ClientConfig = defaultDecodedKibanaClientConfig kibanaHeadersConfig.Kibana.Enabled = true - kibanaHeadersConfig.Kibana.Headers = map[string]string{"foo": "bar"} + kibanaHeadersConfig.Kibana.Headers = map[string]string{ + "foo": "bar", + "Elastic-Api-Version": "2023-10-31", + } responseHeadersConfig := DefaultConfig() responseHeadersConfig.ResponseHeaders = map[string][]string{ diff --git a/internal/beater/otlp/clientmetadata.go b/internal/beater/otlp/clientmetadata.go index 9dbddb1b6ba..fe5a72aa3c7 100644 --- a/internal/beater/otlp/clientmetadata.go +++ b/internal/beater/otlp/clientmetadata.go @@ -45,7 +45,7 @@ func SetClientMetadata(ctx context.Context, batch *modelpb.Batch) error { if event.GetSource().GetIp() == nil { if tcpAddr, ok := clientMetadata.SourceAddr.(*net.TCPAddr); ok { if event.Source == nil { - event.Source = &modelpb.Source{} + event.Source = modelpb.SourceFromVTPool() } sourceAddrPort := tcpAddr.AddrPort() event.Source.Ip = modelpb.Addr2IP(sourceAddrPort.Addr().Unmap()) @@ -54,14 +54,13 @@ func SetClientMetadata(ctx context.Context, batch *modelpb.Batch) error { } if event.GetClient().GetIp() == nil && clientMetadata.ClientIP.IsValid() { if event.Client == nil { - event.Client = &modelpb.Client{} + event.Client = modelpb.ClientFromVTPool() } event.Client.Ip = modelpb.Addr2IP(clientMetadata.ClientIP) } if clientMetadata.SourceNATIP.IsValid() { - event.Source.Nat = &modelpb.NAT{ - Ip: modelpb.Addr2IP(clientMetadata.SourceNATIP), - } + event.Source.Nat = modelpb.NATFromVTPool() + event.Source.Nat.Ip = modelpb.Addr2IP(clientMetadata.SourceNATIP) } } } diff --git a/internal/beater/processors.go b/internal/beater/processors.go index c81d9c6c4a8..02acc59b351 100644 --- a/internal/beater/processors.go +++ b/internal/beater/processors.go @@ -74,7 +74,7 @@ func newObserverBatchProcessor() modelpb.ProcessBatchFunc { return func(ctx context.Context, b *modelpb.Batch) error { for i := range *b { if (*b)[i].Observer == nil { - (*b)[i].Observer = &modelpb.Observer{} + (*b)[i].Observer = modelpb.ObserverFromVTPool() } observer := (*b)[i].Observer observer.Hostname = hostname diff --git a/internal/model/modelprocessor/libraryframe.go b/internal/model/modelprocessor/libraryframe.go index 9d33e8ff767..7e7a803053b 100644 --- a/internal/model/modelprocessor/libraryframe.go +++ b/internal/model/modelprocessor/libraryframe.go @@ -68,7 +68,7 @@ func (s SetLibraryFrame) processException(ctx context.Context, exception *modelp func (s SetLibraryFrame) processStacktraceFrames(ctx context.Context, frames ...*modelpb.StacktraceFrame) { for _, frame := range frames { if frame.Original == nil { - frame.Original = &modelpb.Original{} + frame.Original = modelpb.OriginalFromVTPool() } frame.Original.LibraryFrame = frame.LibraryFrame frame.LibraryFrame = frame.Filename != "" && s.Pattern.MatchString(frame.Filename) || diff --git a/internal/r8/deobfuscator.go b/internal/r8/deobfuscator.go index e56eeaf0f65..8249e6674c4 100644 --- a/internal/r8/deobfuscator.go +++ b/internal/r8/deobfuscator.go @@ -107,7 +107,7 @@ func resolveMappings(types map[string]StacktraceType, mapReader io.Reader) error for _, frames := range stacktraceType.methods { for _, frame := range frames { if frame.Original == nil { - frame.Original = &modelpb.Original{} + frame.Original = modelpb.OriginalFromVTPool() } // Multiple frames might point to the same class, so we need to deobfuscate the class name for them all. frame.Original.Classname = obfuscatedName diff --git a/internal/sourcemap/processor.go b/internal/sourcemap/processor.go index 99889e2741a..6b2174da390 100644 --- a/internal/sourcemap/processor.go +++ b/internal/sourcemap/processor.go @@ -135,7 +135,7 @@ func (p BatchProcessor) processStacktraceFrame( } if frame.Original == nil { - frame.Original = &modelpb.Original{} + frame.Original = modelpb.OriginalFromVTPool() } // Store original source information. diff --git a/internal/version/version.go b/internal/version/version.go index 511dac3b8eb..1e734dbbe66 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -18,4 +18,4 @@ package version // Version holds the APM Server version. -const Version = "8.11.0" +const Version = "8.12.0" diff --git a/systemtest/approvals/TestIntake/Errors.approved.json b/systemtest/approvals/TestIntake/Errors.approved.json index 248b4bfe4fe..edd7bb0cec7 100644 --- a/systemtest/approvals/TestIntake/Errors.approved.json +++ b/systemtest/approvals/TestIntake/Errors.approved.json @@ -10,13 +10,16 @@ }, "client": { "geo": { + "city_name": "Sheridan", "continent_name": "North America", "country_iso_code": "US", "country_name": "United States", "location": { - "lat": 37.751, - "lon": -97.822 - } + "lat": 40.1405, + "lon": -86.2209 + }, + "region_iso_code": "US-IN", + "region_name": "Indiana" }, "ip": "12.53.12.1" }, diff --git a/systemtest/approvals/TestIntake/Events.approved.json b/systemtest/approvals/TestIntake/Events.approved.json index fde852592c7..6ed0ab137e6 100644 --- a/systemtest/approvals/TestIntake/Events.approved.json +++ b/systemtest/approvals/TestIntake/Events.approved.json @@ -565,13 +565,16 @@ }, "client": { "geo": { + "city_name": "Sheridan", "continent_name": "North America", "country_iso_code": "US", "country_name": "United States", "location": { - "lat": 37.751, - "lon": -97.822 - } + "lat": 40.1405, + "lon": -86.2209 + }, + "region_iso_code": "US-IN", + "region_name": "Indiana" }, "ip": "12.53.12.1" }, diff --git a/systemtest/approvals/TestIntake/Transactions.approved.json b/systemtest/approvals/TestIntake/Transactions.approved.json index 9e9d66f159b..6625df83565 100644 --- a/systemtest/approvals/TestIntake/Transactions.approved.json +++ b/systemtest/approvals/TestIntake/Transactions.approved.json @@ -307,13 +307,16 @@ }, "client": { "geo": { + "city_name": "Sheridan", "continent_name": "North America", "country_iso_code": "US", "country_name": "United States", "location": { - "lat": 37.751, - "lon": -97.822 - } + "lat": 40.1405, + "lon": -86.2209 + }, + "region_iso_code": "US-IN", + "region_name": "Indiana" }, "ip": "12.53.12.1" }, diff --git a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json index 801011ce849..594dbe6642d 100644 --- a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json +++ b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json @@ -9,13 +9,16 @@ }, "client": { "geo": { + "city_name": "Sheridan", "continent_name": "North America", "country_iso_code": "US", "country_name": "United States", "location": { - "lat": 37.751, - "lon": -97.822 - } + "lat": 40.1405, + "lon": -86.2209 + }, + "region_iso_code": "US-IN", + "region_name": "Indiana" }, "ip": "12.53.12.1" }, diff --git a/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json b/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json new file mode 100644 index 00000000000..7c29b81a2bf --- /dev/null +++ b/systemtest/approvals/TestOTLPGRPCLogsClientIP.approved.json @@ -0,0 +1,64 @@ +{ + "events": [ + { + "@timestamp": "1970-01-01T00:00:01.000Z", + "agent": { + "name": "android/java", + "version": "unknown" + }, + "client": { + "geo": { + "continent_name": "Europe", + "country_iso_code": "ES", + "country_name": "Spain", + "location": { + "lat": 40.4172, + "lon": -3.684 + } + }, + "ip": "195.55.79.118" + }, + "data_stream": { + "dataset": "apm.app.unknown", + "namespace": "default", + "type": "logs" + }, + "event": { + "severity": 9 + }, + "labels": { + "key": "value" + }, + "log": { + "level": "Info" + }, + "message": "a log message", + "numeric_labels": { + "numeric_key": 1234 + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "service": { + "language": { + "name": "java" + }, + "name": "unknown" + }, + "source": { + "ip": "195.55.79.118", + "nat": { + "ip": "127.0.0.1" + } + }, + "span": { + "id": "0200000000000000" + }, + "trace": { + "id": "01000000000000000000000000000000" + } + } + ] +} diff --git a/systemtest/otlp_test.go b/systemtest/otlp_test.go index 3f7956f82a4..2781af2eb6b 100644 --- a/systemtest/otlp_test.go +++ b/systemtest/otlp_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" + "google.golang.org/grpc/metadata" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -453,6 +454,63 @@ func TestOTLPRateLimit(t *testing.T) { assert.Equal(t, "traces export: rpc error: code = ResourceExhausted desc = rate limit exceeded", errStatus.Message()) } +func TestOTLPGRPCLogsClientIP(t *testing.T) { + systemtest.CleanupElasticsearch(t) + srv := apmservertest.NewUnstartedServerTB(t) + err := srv.Start() + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Override local IP address to be found in "GeoLite2-City.mmdb". + md := metadata.New(map[string]string{"X-Forwarded-For": "195.55.79.118"}) + ctx = metadata.NewOutgoingContext(ctx, md) + + conn, err := grpc.Dial(serverAddr(srv), grpc.WithInsecure(), grpc.WithBlock(), grpc.WithDefaultCallOptions(grpc.UseCompressor("gzip"))) + require.NoError(t, err) + defer conn.Close() + + logsClient := plogotlp.NewGRPCClient(conn) + + logs := newMobileLogs("a log message") + _, err = logsClient.Export(ctx, plogotlp.NewExportRequestFromLogs(logs)) + require.NoError(t, err) + + result := estest.ExpectDocs(t, systemtest.Elasticsearch, "logs-apm*", nil) + approvaltest.ApproveEvents(t, t.Name(), result.Hits.Hits) +} + +func newMobileLogs(body interface{}) plog.Logs { + logs := plog.NewLogs() + resourceLogs := logs.ResourceLogs().AppendEmpty() + resourceAttrs := logs.ResourceLogs().At(0).Resource().Attributes() + resourceAttrs.PutStr(semconv.AttributeTelemetrySDKLanguage, "java") + resourceAttrs.PutStr(semconv.AttributeTelemetrySDKName, "android") + + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + otelLog := scopeLogs.LogRecords().AppendEmpty() + otelLog.SetTraceID(pcommon.TraceID{1}) + otelLog.SetSpanID(pcommon.SpanID{2}) + otelLog.SetSeverityNumber(plog.SeverityNumberInfo) + otelLog.SetSeverityText("Info") + otelLog.SetTimestamp(pcommon.NewTimestampFromTime(time.Unix(1, 0))) + otelLog.Attributes().PutStr("key", "value") + otelLog.Attributes().PutDouble("numeric_key", 1234) + + switch b := body.(type) { + case string: + otelLog.Body().SetStr(b) + case int: + otelLog.Body().SetInt(int64(b)) + case float64: + otelLog.Body().SetDouble(float64(b)) + case bool: + otelLog.Body().SetBool(b) + } + return logs +} + func newOTLPTraceExporter(t testing.TB, srv *apmservertest.Server, options ...otlptracegrpc.Option) *otlptrace.Exporter { options = append(options, otlptracegrpc.WithEndpoint(serverAddr(srv)), otlptracegrpc.WithInsecure()) exporter, err := otlptracegrpc.New(context.Background(), options...) diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index c3f01d12802..ef3cff67d1f 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.11.0-abc4d077-SNAPSHOT + version: 8.11.0-4d413233-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index c436e05b312..6e6d5df63c0 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.11.0-abc4d077-SNAPSHOT + version: 8.11.0-4d413233-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index f1d0bbb8824..f59a3b57bde 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.11.0-abc4d077-SNAPSHOT + version: 8.11.0-4d413233-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch