Merge branch 'main' into dependabot/docker/packaging/docker/ubi9/ubi-…

…micro-a22fffe

.github/workflows/benchmarks.yml

@@ -8,6 +8,16 @@ on:
         required: false
         type: boolean
         default: false
+      enableTailSampling:
+        description: 'Enable tail-based sampling on the APM server'
+        required: false
+        type: boolean
+        default: false
+      tailSamplingStorageLimit:
+        description: 'Storage size limit of tail-based sampling on the APM server, defaults to 10GB'
+        required: false
+        type: string
+        default: "10GB"
       profile:
         description: 'The system profile used to run the benchmarks'
         required: false
@@ -52,12 +62,14 @@ jobs:
       TF_VAR_private_key: ./id_rsa_terraform
       TF_VAR_public_key: ./id_rsa_terraform.pub
       TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
+      TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling || 'false' }} # set the default again otherwise schedules won't work
+      TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work
       RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
       TF_VAR_BUILD_ID: ${{ github.run_id }}
       TF_VAR_ENVIRONMENT: ci
       TF_VAR_REPO: ${{ github.repository }}
-      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
+      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }},enable_tail_sampling=${{ inputs.enableTailSampling }}
       GOBENCH_PASSWORD: ${{ secrets.GOBENCH_PASSWORD }}
       GOBENCH_USERNAME: ${{ secrets.GOBENCH_USERNAME }}
       GOBENCH_HOST: ${{ secrets.GOBENCH_HOST }}

NOTICE.txt

@@ -604,11 +604,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20241213104605-93b018a83621
+Version: v7.0.0-alpha2.0.20241216091513-d508a408b0e9
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20241213104605-93b018a83621/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20241216091513-d508a408b0e9/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of

changelogs/8.15.asciidoc

@@ -101,9 +101,10 @@ https://github.com/elastic/apm-server/compare/v8.14.3\...v8.15.0[View commits]
 
 - Add `elasticsearch.flushed.uncompressed.bytes` metric {pull}13155[13155]
 - APM Server now relies on the Elasticsearch apm-data plugin's index templates, removing the requirement to install the APM integration package {pull}12066[12066]
+- Switch the default lifecycle management to leverage DSL instead of ILM, the change is executed as a part of Elasticsearch apm-data plugin migration {pull}12066[12066]
 - Upgraded bundled APM Java agent attacher CLI to version 1.50.0 {pull}13326[13326]
 - Enable Kibana curated UIs to work with hostmetrics from OpenTelemetry's https://pkg.go.dev/go.opentelemetry.io/collector/receiver/hostmetricsreceiver[hostmetricsreceiver] {pull}13196[13196]
 - Add require data stream to bulk index requests {pull}13398[13398]
 - Support self-instrumentation when in managed mode by getting tracing configs via reloader {pull}13514[13514] {pull}13653[13653] {pull}13691[13691] {pull}13790[13790]
 - Add mapping for OpenTelemetry attribute `messaging.destination.name` to derive `service.target` correctly {pull}13472[13472]
-- APM Server now automatically retries document-level 429s from Elasticsearch to avoid dropping data. `output.elasticsearch.max_retries` now controls both request-level and document-level retries, and defaults to `3`. {pull}13620[13620]
+- APM Server now automatically retries document-level 429s from Elasticsearch to avoid dropping data. `output.elasticsearch.max_retries` now controls both request-level and document-level retries, and defaults to `3`. {pull}13620[13620]

changelogs/8.16.asciidoc

@@ -1,9 +1,36 @@
 [[apm-release-notes-8.16]]
 == APM version 8.16
 
+* <<apm-release-notes-8.16.2>>
 * <<apm-release-notes-8.16.1>>
 * <<apm-release-notes-8.16.0>>
 
+[float]
+[[apm-release-notes-8.16.2]]
+=== APM version 8.16.2
+
+https://github.com/elastic/apm-server/compare/v8.16.1\...v8.16.2[View commits]
+
+[float]
+==== Bug fixes
+
+- Surface config parsing error under EA managed mode by logging and marking EA input unit as failed {pull}14574[14574]
+- Remove unnecessary hot reload under EA managed mode when apm tracing config is nil {pull}14865[14865]
+
+[float]
+==== Breaking Changes
+
+[float]
+==== Deprecations
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+
+- In this release we've introduced an image based on the hardened https://wolfi.dev/[Wolfi] image to provide additional security to our self-managed customers, and improve our supply chain security posture.
+
 [float]
 [[apm-release-notes-8.16.1]]
 === APM version 8.16.1

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v1.1.0
 	github.com/elastic/apm-data v1.14.4
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241213104605-93b018a83621
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241216091513-d508a408b0e9
 	github.com/elastic/elastic-agent-client/v7 v7.17.0
 	github.com/elastic/elastic-agent-libs v0.17.4
 	github.com/elastic/elastic-agent-system-metrics v0.11.5

go.sum

@@ -136,8 +136,8 @@ github.com/elastic/apm-aggregation v1.1.0 h1:eTHGd5w99JoRFJ763MJP6nUX4hkkeSil3KW
 github.com/elastic/apm-aggregation v1.1.0/go.mod h1:YBQ77Jt7vOy2/7w4q5SsEifhc0V02dJxRcyyeC6HsvQ=
 github.com/elastic/apm-data v1.14.4 h1:Q84GljGLbPEnUCcF7i/CWFx6Ey1Rx+ffvSUyEy/k0R4=
 github.com/elastic/apm-data v1.14.4/go.mod h1:MctdMZ5LBx07m0TjJTdh80nU2ef6ABcoyJWg1E14/kg=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241213104605-93b018a83621 h1:1fcMJDnMDx25sAQM7+YC5DeS+ovVWRkjWAUgR04pjDE=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241213104605-93b018a83621/go.mod h1:JZIuztvKeiuHUM8zvfQ24wz6nJPPOM1DVqu9FHu9a0Q=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241216091513-d508a408b0e9 h1:A+n1lfJyAxMQwWHq1nOkOMHk3rMOxFqgtbijT1Uh0aM=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241216091513-d508a408b0e9/go.mod h1:JZIuztvKeiuHUM8zvfQ24wz6nJPPOM1DVqu9FHu9a0Q=
 github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0JojNasPgYIA47gpuuns=
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.17.0 h1:TPLrEHF4kJ3RkmQzZPffrniY4WeW4vriHZbOAzM1hFo=

systemtest/approvals/TestRUMXForwardedFor.approved.json

@@ -13,7 +13,7 @@
             "5.5.0"
         ],
         "client.geo.city_name": [
-            "Perth"
+            "dynamic"
         ],
         "client.geo.continent_name": [
             "Oceania"
@@ -28,10 +28,10 @@
             "dynamic"
         ],
         "client.geo.region_iso_code": [
-            "AU-WA"
+            "dynamic"
         ],
         "client.geo.region_name": [
-            "Western Australia"
+            "dynamic"
         ],
         "client.ip": [
             "220.244.41.16"
@@ -132,7 +132,7 @@
             "5.5.0"
         ],
         "client.geo.city_name": [
-            "Perth"
+            "dynamic"
         ],
         "client.geo.continent_name": [
             "Oceania"
@@ -147,10 +147,10 @@
             "dynamic"
         ],
         "client.geo.region_iso_code": [
-            "AU-WA"
+            "dynamic"
         ],
         "client.geo.region_name": [
-            "Western Australia"
+            "dynamic"
         ],
         "client.ip": [
             "220.244.41.16"

systemtest/benchtest/expvar/expvar.go

@@ -36,6 +36,7 @@ type expvar struct {
 	LibbeatStats
 	ElasticResponseStats
 	OTLPResponseStats
+	TailSamplingStats
 
 	// UncompressedBytes holds the number of bytes of uncompressed
 	// data that the server has read from the Elastic APM events
@@ -72,6 +73,11 @@ type LibbeatStats struct {
 	RSSMemoryBytes int64 `json:"beat.memstats.rss"`
 }
 
+type TailSamplingStats struct {
+	TBSLsmSize  int64 `json:"apm-server.sampling.tail.storage.lsm_size"`
+	TBSVlogSize int64 `json:"apm-server.sampling.tail.storage.value_log_size"`
+}
+
 func queryExpvar(ctx context.Context, out *expvar, srv string) error {
 	req, err := http.NewRequest("GET", srv+"/debug/vars", nil)
 	if err != nil {
@@ -113,6 +119,7 @@ func queryExpvar(ctx context.Context, out *expvar, srv string) error {
 		aggregateResponseStats(s.ElasticResponseStats, &result.ElasticResponseStats)
 		aggregateOTLPResponseStats(s.OTLPResponseStats, &result.OTLPResponseStats)
 		aggregateLibbeatStats(s.LibbeatStats, &result.LibbeatStats)
+		aggregateTailSamplingStats(s.TailSamplingStats, &result.TailSamplingStats)
 		result.UncompressedBytes += s.UncompressedBytes
 		result.AvailableBulkRequests += s.AvailableBulkRequests
 	}
@@ -205,3 +212,8 @@ func aggregateOTLPResponseStats(from OTLPResponseStats, to *OTLPResponseStats) {
 	to.ErrorOTLPTracesResponses += from.ErrorOTLPTracesResponses
 	to.ErrorOTLPMetricsResponses += from.ErrorOTLPMetricsResponses
 }
+
+func aggregateTailSamplingStats(from TailSamplingStats, to *TailSamplingStats) {
+	to.TBSLsmSize += from.TBSLsmSize
+	to.TBSVlogSize += from.TBSVlogSize
+}

systemtest/benchtest/expvar/metrics.go

@@ -47,6 +47,8 @@ const (
 	ErrorElasticResponses
 	ErrorOTLPTracesResponses
 	ErrorOTLPMetricsResponses
+	TBSLsmSize
+	TBSVlogSize
 )
 
 type AggregateStats struct {
@@ -164,6 +166,8 @@ func (c *Collector) accumulate(e expvar) {
 	c.processMetric(MemBytes, int64(e.TotalAlloc))
 	c.processMetric(HeapAlloc, int64(e.HeapAlloc))
 	c.processMetric(HeapObjects, int64(e.HeapObjects))
+	c.processMetric(TBSLsmSize, e.TBSLsmSize)
+	c.processMetric(TBSVlogSize, e.TBSVlogSize)
 }
 
 func (c *Collector) processMetric(m Metric, val int64) {

systemtest/benchtest/main.go

@@ -122,6 +122,8 @@ func addExpvarMetrics(result *testing.BenchmarkResult, collector *expvar.Collect
 		result.Extra["max_heap_alloc"] = float64(collector.Get(expvar.HeapAlloc).Max)
 		result.Extra["max_heap_objects"] = float64(collector.Get(expvar.HeapObjects).Max)
 		result.Extra["mean_available_indexers"] = float64(collector.Get(expvar.AvailableBulkRequests).Mean)
+		result.Extra["tbs_lsm_size"] = float64(collector.Get(expvar.TBSLsmSize).Max)
+		result.Extra["tbs_vlog_size"] = float64(collector.Get(expvar.TBSVlogSize).Max)
 	}
 
 	// Record the number of error responses returned by the server: lower is better.

systemtest/benchtest/main_test.go

@@ -141,6 +141,8 @@ func TestAddExpvarMetrics(t *testing.T) {
 				`"apm-server.processor.span.transformations": 5`,
 				`"apm-server.processor.metric.transformations": 9`,
 				`"apm-server.processor.error.transformations": 3`,
+				`"apm-server.sampling.tail.storage.lsm_size": 10`,
+				`"apm-server.sampling.tail.storage.value_log_size": 11`,
 				`"beat.runtime.goroutines": 4`,
 				`"beat.memstats.rss": 1048576`,
 				`"output.elasticsearch.bulk_requests.available": 0`,
@@ -165,6 +167,8 @@ func TestAddExpvarMetrics(t *testing.T) {
 				"max_heap_objects":        102,
 				"mean_available_indexers": 0,
 				"error_responses/sec":     1,
+				"tbs_lsm_size":            10,
+				"tbs_vlog_size":           11,
 			},
 		},
 	}

systemtest/rum_test.go

@@ -78,7 +78,10 @@ func TestRUMXForwardedFor(t *testing.T) {
 		"source.port",
 		// Do not assert the exact contents of the location field since they may change
 		// slightly depending on the IP lookup.
+		"client.geo.city_name",
 		"client.geo.location",
+		"client.geo.region_iso_code",
+		"client.geo.region_name",
 	)
 }
 

testing/benchmark/main.tf

@@ -97,12 +97,14 @@ module "ec_deployment" {
   deployment_template    = var.deployment_template
   deployment_name_prefix = local.name_prefix
 
-  apm_server_size       = var.apm_server_size
-  apm_server_zone_count = var.apm_server_zone_count
-  apm_index_shards      = var.apm_shards
-  drop_pipeline         = var.drop_pipeline
-  apm_server_expvar     = true
-  apm_server_pprof      = true
+  apm_server_size                        = var.apm_server_size
+  apm_server_zone_count                  = var.apm_server_zone_count
+  apm_index_shards                       = var.apm_shards
+  drop_pipeline                          = var.drop_pipeline
+  apm_server_expvar                      = true
+  apm_server_pprof                       = true
+  apm_server_tail_sampling               = var.apm_server_tail_sampling
+  apm_server_tail_sampling_storage_limit = var.apm_server_tail_sampling_storage_limit
 
   elasticsearch_size              = var.elasticsearch_size
   elasticsearch_zone_count        = var.elasticsearch_zone_count

testing/benchmark/variables.tf

@@ -92,6 +92,18 @@ variable "drop_pipeline" {
   type        = bool
 }
 
+variable "apm_server_tail_sampling" {
+  default     = false
+  description = "Whether or not to enable APM Server tail-based sampling. Defaults to false"
+  type        = bool
+}
+
+variable "apm_server_tail_sampling_storage_limit" {
+  default     = "10GB"
+  description = "Storage size limit of APM Server tail-based sampling. Defaults to 10GB"
+  type        = string
+}
+
 # Standalone
 
 variable "apm_server_bin_path" {

testing/infra/terraform/modules/ec_deployment/README.md

@@ -28,12 +28,12 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | [ec_deployment.deployment](https://registry.terraform.io/providers/elastic/ec/0.5.1/docs/resources/deployment) | resource |
 | [local_file.custom_apm_integration_pkg](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.drop_pipeline](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
-| [local_file.enable_expvar](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.enable_features](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.secret_token](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.shard_settings](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [null_resource.custom_apm_integration_pkg](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.drop_pipeline](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
-| [null_resource.enable_expvar](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
+| [null_resource.enable_features](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.secret_token](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.shard_settings](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [ec_stack.deployment_version](https://registry.terraform.io/providers/elastic/ec/0.5.1/docs/data-sources/stack) | data source |
@@ -47,12 +47,14 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | <a name="input_apm_server_expvar"></a> [apm\_server\_expvar](#input\_apm\_server\_expvar) | Whether or not to enable APM Server's expvar endpoint. Defaults to false | `bool` | `false` | no |
 | <a name="input_apm_server_pprof"></a> [apm\_server\_pprof](#input\_apm\_server\_pprof) | Whether or not to enable APM Server's pprof endpoint. Defaults to false | `bool` | `false` | no |
 | <a name="input_apm_server_size"></a> [apm\_server\_size](#input\_apm\_server\_size) | Optional apm server instance size | `string` | `"1g"` | no |
+| <a name="input_apm_server_tail_sampling"></a> [apm\_server\_tail\_sampling](#input\_apm\_server\_tail\_sampling) | Whether or not to enable APM Server tail-based sampling. Defaults to false | `bool` | `false` | no |
+| <a name="input_apm_server_tail_sampling_storage_limit"></a> [apm\_server\_tail\_sampling\_storage\_limit](#input\_apm\_server\_tail\_sampling\_storage\_limit) | Storage size limit of APM Server tail-based sampling. Defaults to 10GB | `string` | `"10GB"` | no |
 | <a name="input_apm_server_zone_count"></a> [apm\_server\_zone\_count](#input\_apm\_server\_zone\_count) | Optional apm server zone count | `number` | `1` | no |
 | <a name="input_custom_apm_integration_pkg_path"></a> [custom\_apm\_integration\_pkg\_path](#input\_custom\_apm\_integration\_pkg\_path) | Path to the zipped custom APM integration package, if empty custom apm integration pkg is not installed | `string` | `""` | no |
 | <a name="input_deployment_name_prefix"></a> [deployment\_name\_prefix](#input\_deployment\_name\_prefix) | Optional ESS or ECE region. Defaults to GCP US West 2 (Los Angeles) | `string` | `"apmserver"` | no |
 | <a name="input_deployment_template"></a> [deployment\_template](#input\_deployment\_template) | Optional deployment template. Defaults to the CPU optimized template for GCP | `string` | `"gcp-cpu-optimized"` | no |
-| <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Optional docker image overrides. The full map needs to be specified | `map(string)` | <pre>{<br>  "apm": "docker.elastic.co/cloud-release/elastic-agent-cloud",<br>  "elasticsearch": "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",<br>  "kibana": "docker.elastic.co/cloud-release/kibana-cloud"<br>}</pre> | no |
-| <a name="input_docker_image_tag_override"></a> [docker\_image\_tag\_override](#input\_docker\_image\_tag\_override) | Optional docker image tag overrides, The full map needs to be specified | `map(string)` | <pre>{<br>  "apm": "",<br>  "elasticsearch": "",<br>  "kibana": ""<br>}</pre> | no |
+| <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Optional docker image overrides. The full map needs to be specified | `map(string)` | <pre>{<br/>  "apm": "docker.elastic.co/cloud-release/elastic-agent-cloud",<br/>  "elasticsearch": "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",<br/>  "kibana": "docker.elastic.co/cloud-release/kibana-cloud"<br/>}</pre> | no |
+| <a name="input_docker_image_tag_override"></a> [docker\_image\_tag\_override](#input\_docker\_image\_tag\_override) | Optional docker image tag overrides, The full map needs to be specified | `map(string)` | <pre>{<br/>  "apm": "",<br/>  "elasticsearch": "",<br/>  "kibana": ""<br/>}</pre> | no |
 | <a name="input_drop_pipeline"></a> [drop\_pipeline](#input\_drop\_pipeline) | Whether or not to install an Elasticsearch ingest pipeline to drop all incoming APM documents. Defaults to false | `bool` | `false` | no |
 | <a name="input_elasticsearch_autoscale"></a> [elasticsearch\_autoscale](#input\_elasticsearch\_autoscale) | Optional autoscale the Elasticsearch cluster | `bool` | `false` | no |
 | <a name="input_elasticsearch_dedicated_masters"></a> [elasticsearch\_dedicated\_masters](#input\_elasticsearch\_dedicated\_masters) | Optionally use dedicated masters for the Elasticsearch cluster | `bool` | `false` | no |