Merge branch 'main' into dependabot/go_modules/otel-be5341d2b5

elastic · Dec 20, 2024 · 5614d19 · 5614d19
2 parents e2c859f + ba30c6a
commit 5614d19
Show file tree

Hide file tree

Showing 19 changed files with 160 additions and 95 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -8,6 +8,16 @@ on:
         required: false
         type: boolean
         default: false
+      enableTailSampling:
+        description: 'Enable tail-based sampling on the APM server'
+        required: false
+        type: boolean
+        default: false
+      tailSamplingStorageLimit:
+        description: 'Storage size limit of tail-based sampling on the APM server, defaults to 10GB'
+        required: false
+        type: string
+        default: "10GB"
       profile:
         description: 'The system profile used to run the benchmarks'
         required: false
@@ -52,12 +62,14 @@ jobs:
       TF_VAR_private_key: ./id_rsa_terraform
       TF_VAR_public_key: ./id_rsa_terraform.pub
       TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
+      TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling || 'false' }} # set the default again otherwise schedules won't work
+      TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work
       RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
       TF_VAR_BUILD_ID: ${{ github.run_id }}
       TF_VAR_ENVIRONMENT: ci
       TF_VAR_REPO: ${{ github.repository }}
-      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
+      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }},enable_tail_sampling=${{ inputs.enableTailSampling }}
       GOBENCH_PASSWORD: ${{ secrets.GOBENCH_PASSWORD }}
       GOBENCH_USERNAME: ${{ secrets.GOBENCH_USERNAME }}
       GOBENCH_HOST: ${{ secrets.GOBENCH_HOST }}

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -1704,11 +1704,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-docappender/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-elasticsearch/v8
-Version: v8.16.0
+Version: v8.17.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.16.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.17.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -5706,11 +5706,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
-Version: v0.32.0
+Version: v0.33.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.32.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.33.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 

diff --git a/go.mod b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/elastic/elastic-agent-system-metrics v0.11.5
 	github.com/elastic/gmux v0.3.2
 	github.com/elastic/go-docappender/v2 v2.3.2
-	github.com/elastic/go-elasticsearch/v8 v8.16.0
+	github.com/elastic/go-elasticsearch/v8 v8.17.0
 	github.com/elastic/go-sysinfo v1.15.0
 	github.com/elastic/go-ucfg v0.8.8
 	github.com/go-sourcemap/sourcemap v2.1.4+incompatible
@@ -48,7 +48,7 @@ require (
 	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/zap v1.27.0
 	go.uber.org/zap/exp v0.3.0
-	golang.org/x/net v0.32.0
+	golang.org/x/net v0.33.0
 	golang.org/x/sync v0.10.0
 	golang.org/x/term v0.27.0
 	golang.org/x/time v0.8.0

diff --git a/go.sum b/go.sum
@@ -152,8 +152,8 @@ github.com/elastic/gmux v0.3.2 h1:cb721R+fe/rt/jVNyBP5HDQsEwLD2wSKfPD2Sk6adDk=
 github.com/elastic/gmux v0.3.2/go.mod h1:OD6oYrno+SV3pyl1ArdWCjlExZ+FJOfoSaFqnFeldBQ=
 github.com/elastic/go-docappender/v2 v2.3.2 h1:FJhYgq2DpCaxGaZUquc75dauEzWTWOyWUESAA/h8SPY=
 github.com/elastic/go-docappender/v2 v2.3.2/go.mod h1:5URybRUfmexRMtM/lwvcIRLje3Gsrj15qiiLm41gDrc=
-github.com/elastic/go-elasticsearch/v8 v8.16.0 h1:f7bR+iBz8GTAVhwyFO3hm4ixsz2eMaEy0QroYnXV3jE=
-github.com/elastic/go-elasticsearch/v8 v8.16.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64=
+github.com/elastic/go-elasticsearch/v8 v8.17.0 h1:e9cWksE/Fr7urDRmGPGp47Nsp4/mvNOrU8As1l2HQQ0=
+github.com/elastic/go-elasticsearch/v8 v8.17.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64=
 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt7YZoupKlxZ093qTZYdGou3EhfTF1U0B4=
 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko=
 github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w=
@@ -522,8 +522,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
-golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
 golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=

diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
@@ -30,7 +30,7 @@ RUN sed -i 's/localhost:9200/elasticsearch:9200/' apm-server.yml
 # Build stage 1
 # Copy prepared files from the previous stage and complete the image.
 ################################################################################
-FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:a410623c2b8e9429f9606af821be0231fef2372bd0f5f853fbe9743a0ddf7b34
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:a22fffe0256af00176c8b4f22eec5d8ecb1cb1684d811c33b1f2832fd573260f
 ARG TARGETARCH
 ARG BUILD_DATE
 ARG VERSION

diff --git a/systemtest/approvals/TestRUMXForwardedFor.approved.json b/systemtest/approvals/TestRUMXForwardedFor.approved.json
@@ -13,7 +13,7 @@
             "5.5.0"
         ],
         "client.geo.city_name": [
-            "Perth"
+            "dynamic"
         ],
         "client.geo.continent_name": [
             "Oceania"
@@ -28,10 +28,10 @@
             "dynamic"
         ],
         "client.geo.region_iso_code": [
-            "AU-WA"
+            "dynamic"
         ],
         "client.geo.region_name": [
-            "Western Australia"
+            "dynamic"
         ],
         "client.ip": [
             "220.244.41.16"
@@ -132,7 +132,7 @@
             "5.5.0"
         ],
         "client.geo.city_name": [
-            "Perth"
+            "dynamic"
         ],
         "client.geo.continent_name": [
             "Oceania"
@@ -147,10 +147,10 @@
             "dynamic"
         ],
         "client.geo.region_iso_code": [
-            "AU-WA"
+            "dynamic"
         ],
         "client.geo.region_name": [
-            "Western Australia"
+            "dynamic"
         ],
         "client.ip": [
             "220.244.41.16"

diff --git a/systemtest/benchtest/expvar/expvar.go b/systemtest/benchtest/expvar/expvar.go
@@ -36,6 +36,7 @@ type expvar struct {
 	LibbeatStats
 	ElasticResponseStats
 	OTLPResponseStats
+	TailSamplingStats
 
 	// UncompressedBytes holds the number of bytes of uncompressed
 	// data that the server has read from the Elastic APM events
@@ -72,6 +73,11 @@ type LibbeatStats struct {
 	RSSMemoryBytes int64 `json:"beat.memstats.rss"`
 }
 
+type TailSamplingStats struct {
+	TBSLsmSize  int64 `json:"apm-server.sampling.tail.storage.lsm_size"`
+	TBSVlogSize int64 `json:"apm-server.sampling.tail.storage.value_log_size"`
+}
+
 func queryExpvar(ctx context.Context, out *expvar, srv string) error {
 	req, err := http.NewRequest("GET", srv+"/debug/vars", nil)
 	if err != nil {
@@ -113,6 +119,7 @@ func queryExpvar(ctx context.Context, out *expvar, srv string) error {
 		aggregateResponseStats(s.ElasticResponseStats, &result.ElasticResponseStats)
 		aggregateOTLPResponseStats(s.OTLPResponseStats, &result.OTLPResponseStats)
 		aggregateLibbeatStats(s.LibbeatStats, &result.LibbeatStats)
+		aggregateTailSamplingStats(s.TailSamplingStats, &result.TailSamplingStats)
 		result.UncompressedBytes += s.UncompressedBytes
 		result.AvailableBulkRequests += s.AvailableBulkRequests
 	}
@@ -205,3 +212,8 @@ func aggregateOTLPResponseStats(from OTLPResponseStats, to *OTLPResponseStats) {
 	to.ErrorOTLPTracesResponses += from.ErrorOTLPTracesResponses
 	to.ErrorOTLPMetricsResponses += from.ErrorOTLPMetricsResponses
 }
+
+func aggregateTailSamplingStats(from TailSamplingStats, to *TailSamplingStats) {
+	to.TBSLsmSize += from.TBSLsmSize
+	to.TBSVlogSize += from.TBSVlogSize
+}
diff --git a/systemtest/benchtest/expvar/metrics.go b/systemtest/benchtest/expvar/metrics.go
@@ -47,6 +47,8 @@ const (
 	ErrorElasticResponses
 	ErrorOTLPTracesResponses
 	ErrorOTLPMetricsResponses
+	TBSLsmSize
+	TBSVlogSize
 )
 
 type AggregateStats struct {
@@ -164,6 +166,8 @@ func (c *Collector) accumulate(e expvar) {
 	c.processMetric(MemBytes, int64(e.TotalAlloc))
 	c.processMetric(HeapAlloc, int64(e.HeapAlloc))
 	c.processMetric(HeapObjects, int64(e.HeapObjects))
+	c.processMetric(TBSLsmSize, e.TBSLsmSize)
+	c.processMetric(TBSVlogSize, e.TBSVlogSize)
 }
 
 func (c *Collector) processMetric(m Metric, val int64) {

diff --git a/systemtest/benchtest/main.go b/systemtest/benchtest/main.go
@@ -122,6 +122,8 @@ func addExpvarMetrics(result *testing.BenchmarkResult, collector *expvar.Collect
 		result.Extra["max_heap_alloc"] = float64(collector.Get(expvar.HeapAlloc).Max)
 		result.Extra["max_heap_objects"] = float64(collector.Get(expvar.HeapObjects).Max)
 		result.Extra["mean_available_indexers"] = float64(collector.Get(expvar.AvailableBulkRequests).Mean)
+		result.Extra["tbs_lsm_size"] = float64(collector.Get(expvar.TBSLsmSize).Max)
+		result.Extra["tbs_vlog_size"] = float64(collector.Get(expvar.TBSVlogSize).Max)
 	}
 
 	// Record the number of error responses returned by the server: lower is better.

diff --git a/systemtest/benchtest/main_test.go b/systemtest/benchtest/main_test.go
@@ -141,6 +141,8 @@ func TestAddExpvarMetrics(t *testing.T) {
 				`"apm-server.processor.span.transformations": 5`,
 				`"apm-server.processor.metric.transformations": 9`,
 				`"apm-server.processor.error.transformations": 3`,
+				`"apm-server.sampling.tail.storage.lsm_size": 10`,
+				`"apm-server.sampling.tail.storage.value_log_size": 11`,
 				`"beat.runtime.goroutines": 4`,
 				`"beat.memstats.rss": 1048576`,
 				`"output.elasticsearch.bulk_requests.available": 0`,
@@ -165,6 +167,8 @@ func TestAddExpvarMetrics(t *testing.T) {
 				"max_heap_objects":        102,
 				"mean_available_indexers": 0,
 				"error_responses/sec":     1,
+				"tbs_lsm_size":            10,
+				"tbs_vlog_size":           11,
 			},
 		},
 	}

diff --git a/systemtest/go.mod b/systemtest/go.mod
@@ -19,21 +19,21 @@ require (
 	go.elastic.co/fastjson v1.4.0
 	go.opentelemetry.io/collector/pdata v1.21.0
 	go.opentelemetry.io/collector/semconv v0.115.0
-	go.opentelemetry.io/otel v1.32.0
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0
-	go.opentelemetry.io/otel/metric v1.32.0
-	go.opentelemetry.io/otel/sdk v1.32.0
-	go.opentelemetry.io/otel/sdk/metric v1.32.0
-	go.opentelemetry.io/otel/trace v1.32.0
+	go.opentelemetry.io/otel v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0
+	go.opentelemetry.io/otel/metric v1.33.0
+	go.opentelemetry.io/otel/sdk v1.33.0
+	go.opentelemetry.io/otel/sdk/metric v1.33.0
+	go.opentelemetry.io/otel/trace v1.33.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/sync v0.10.0
 	golang.org/x/sys v0.28.0
 	golang.org/x/time v0.8.0
-	google.golang.org/grpc v1.68.1
+	google.golang.org/grpc v1.69.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -59,7 +59,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
@@ -89,14 +89,15 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
 	google.golang.org/protobuf v1.35.2 // indirect
 	howett.net/plist v1.0.0 // indirect
 )