Merge branch 'main' into dependabot/go_modules/otel-b79dbb9962

NOTICE.txt

@@ -362,11 +362,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/apm-data
-Version: v1.13.1
+Version: v1.14.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.13.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.14.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20241101151741-249d0dc59f82
+Version: v7.0.0-alpha2.0.20241106142216-cf13781a6fca
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20241101151741-249d0dc59f82/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20241106142216-cf13781a6fca/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -829,11 +829,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.17.0
+Version: v0.17.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -10055,11 +10055,11 @@ limitations under the License.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/opentelemetry-lib
-Version: v0.9.0
+Version: v0.12.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-lib@v0.9.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-lib@v0.12.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

docker-compose.yml

@@ -11,7 +11,7 @@ x-logging: &default-logging
 services:
   elasticsearch:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-263e2f29-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-8f0a2811-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -43,7 +43,7 @@ services:
 
   kibana:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/kibana/kibana:9.0.0-263e2f29-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:9.0.0-8f0a2811-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -63,7 +63,7 @@ services:
 
   metricbeat:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/beats/metricbeat:9.0.0-263e2f29-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:9.0.0-8f0a2811-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

go.mod

@@ -7,10 +7,10 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.4
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v1.1.0
-	github.com/elastic/apm-data v1.13.1
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241101151741-249d0dc59f82
+	github.com/elastic/apm-data v1.14.1
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241106142216-cf13781a6fca
 	github.com/elastic/elastic-agent-client/v7 v7.16.0
-	github.com/elastic/elastic-agent-libs v0.17.0
+	github.com/elastic/elastic-agent-libs v0.17.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.3
 	github.com/elastic/gmux v0.3.2
 	github.com/elastic/go-docappender/v2 v2.3.0
@@ -90,7 +90,7 @@ require (
 	github.com/elastic/go-structform v0.0.10 // indirect
 	github.com/elastic/go-windows v1.0.2 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
-	github.com/elastic/opentelemetry-lib v0.9.0 // indirect
+	github.com/elastic/opentelemetry-lib v0.12.0 // indirect
 	github.com/elastic/pkcs8 v1.0.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/frankban/quicktest v1.14.0 // indirect

go.sum

@@ -130,16 +130,16 @@ github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+Gv
 github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/elastic/apm-aggregation v1.1.0 h1:eTHGd5w99JoRFJ763MJP6nUX4hkkeSil3KWiip7YtJY=
 github.com/elastic/apm-aggregation v1.1.0/go.mod h1:YBQ77Jt7vOy2/7w4q5SsEifhc0V02dJxRcyyeC6HsvQ=
-github.com/elastic/apm-data v1.13.1 h1:vQUWqi9H6QC4FCZ05kILkE+2BeKWw6iT+NXXyadWJm8=
-github.com/elastic/apm-data v1.13.1/go.mod h1:m2FSH7alN07VmtmN10KmgxAa8tgXlJDe/rC0g4FnZUI=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241101151741-249d0dc59f82 h1:bX5zv0QOdSQGI/11vJ+7Rcy50qIt+BsWYyV6C+WMZRI=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241101151741-249d0dc59f82/go.mod h1:eCxJTNQhr9Uqd8AxN5ZmoEmwP3siol1GZ6cwu0CemVI=
+github.com/elastic/apm-data v1.14.1 h1:3G8gJ4wjdd5O6JnxtdiEzVrKuqPT9mVqo03d5i5sRJ4=
+github.com/elastic/apm-data v1.14.1/go.mod h1:AlwPwWh15EWP5yLDm/X40mdw1z/pHdlcTgMUkS9tKmY=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241106142216-cf13781a6fca h1:187f3h510qko1D8eFEXFDYtRKdiTCovEVWWkwj4r+w8=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20241106142216-cf13781a6fca/go.mod h1:AM/24BOEwZhMVAAY1d85Bd1CU6dAZr4XiwWrbIjojn4=
 github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0JojNasPgYIA47gpuuns=
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.17.0 h1:KvgigcnlH74yZGZyKx7tqn9zBzy6Bt+AVhg3PfrxNW0=
-github.com/elastic/elastic-agent-libs v0.17.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-libs v0.17.1 h1:1MXoc1eHGE8hCdVJ9+qiGiZAGeHzT2QBVVzD/oxwqeU=
+github.com/elastic/elastic-agent-libs v0.17.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A=
 github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
@@ -162,8 +162,8 @@ github.com/elastic/go-windows v1.0.2 h1:yoLLsAsV5cfg9FLhZ9EXZ2n2sQFKeDYrHenkcivY
 github.com/elastic/go-windows v1.0.2/go.mod h1:bGcDpBzXgYSqM0Gx3DM4+UxFj300SZLixie9u9ixLM8=
 github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
 github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
-github.com/elastic/opentelemetry-lib v0.9.0 h1:auMnq5zxqnB3eQNNdfz8+mtooy0D8+2ZEusU0sHeDj4=
-github.com/elastic/opentelemetry-lib v0.9.0/go.mod h1:H4ECk7whGn41etgYL4zQWJ2BWN5xwP2A5HUIIwNrM9Y=
+github.com/elastic/opentelemetry-lib v0.12.0 h1:SiCGkT7gLOzkl0wQosQrIWjHWb5eJY18Cm3V3GmdU0o=
+github.com/elastic/opentelemetry-lib v0.12.0/go.mod h1:fdpkzh517xJqSGq3bo/fkdoX/Ag0OoanJoMoIDC3bBk=
 github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA=
 github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0=
 github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 h1:5/RUNg7rkIvayjPhAIoI3v8p45NfWcfWs5DZSElycis=

internal/beatcmd/cmd.go

@@ -47,8 +47,11 @@ func NewRootCommand(beatParams BeatParams) *cobra.Command {
 	// root command is an alias for "run"
 	runCommand := genRunCmd(beatParams)
 	rootCommand := &cobra.Command{
-		Use:  "apm-server",
-		RunE: runCommand.RunE,
+		Use: "apm-server",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfgfile.HandleFlags()
+			return runCommand.RunE(cmd, args)
+		},
 		CompletionOptions: cobra.CompletionOptions{
 			DisableDefaultCmd: true,
 		},

internal/beatcmd/init.go

@@ -19,12 +19,9 @@ package beatcmd
 
 import (
 	cryptorand "crypto/rand"
-	"log"
 	"math"
 	"math/big"
 	"math/rand"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
@@ -55,14 +52,11 @@ func initRand() {
 }
 
 func initFlags() {
-	// For backwards compatibility, convert -flags to --flags.
-	for i, arg := range os.Args[1:] {
-		if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "--") && len(arg) > 2 {
-			os.Args[1+i] = "-" + arg
-		}
-	}
-
-	if err := cfgfile.HandleFlags(); err != nil {
-		log.Fatal(err)
-	}
+	// For backwards compatibility, initialize and convert known -flags to --flags.
+	cfgfile.Initialize()
+	cfgfile.AddAllowedBackwardsCompatibleFlag("v")
+	cfgfile.AddAllowedBackwardsCompatibleFlag("e")
+	cfgfile.AddAllowedBackwardsCompatibleFlag("d")
+	cfgfile.AddAllowedBackwardsCompatibleFlag("environment")
+	cfgfile.ConvertFlagsForBackwardsCompatibility()
 }

systemtest/approvals/TestIntake/Errors.approved.json

@@ -663,6 +663,9 @@
         "error.id": [
             "xFoaabb123FFFFFF"
         ],
+        "error.log.level": [
+            "error"
+        ],
         "error.log.message": [
             "no user found"
         ],

systemtest/approvals/TestIntake/MinimalEvents.approved.json

@@ -30,6 +30,9 @@
         "error.id": [
             "abcdef0123456789"
         ],
+        "error.log.level": [
+            "error"
+        ],
         "error.log.message": [
             "error log message"
         ],

systemtest/approvals/TestJaeger/batch_1.approved.json

@@ -348,9 +348,6 @@
         "observer.version": [
             "dynamic"
         ],
-        "parent.id": [
-            "7be2fd98d0973be3"
-        ],
         "service.language.name": [
             "Go"
         ],

systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json

@@ -42,6 +42,9 @@
         "error.id": [
             "aba2688e033848ce9c4e4005f1caa534"
         ],
+        "error.log.level": [
+            "error"
+        ],
         "error.log.message": [
             "Uncaught Error: log timeout test error"
         ],

systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json

@@ -42,6 +42,9 @@
         "error.id": [
             "aba2688e033848ce9c4e4005f1caa534"
         ],
+        "error.log.level": [
+            "error"
+        ],
         "error.log.message": [
             "Uncaught Error: log timeout test error"
         ],

systemtest/go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/elastic/apm-tools v0.0.0-20240607105915-a4f490dc6959
 	github.com/elastic/go-elasticsearch/v8 v8.15.0
 	github.com/elastic/go-lumber v0.1.1
-	github.com/elastic/go-sysinfo v1.14.2
+	github.com/elastic/go-sysinfo v1.15.0
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/jaegertracing/jaeger v1.62.0

systemtest/go.sum

@@ -43,8 +43,8 @@ github.com/elastic/go-elasticsearch/v8 v8.15.0 h1:IZyJhe7t7WI3NEFdcHnf6IJXqpRf+8
 github.com/elastic/go-elasticsearch/v8 v8.15.0/go.mod h1:HCON3zj4btpqs2N1jjsAy4a/fiAul+YBP00mBH4xik8=
 github.com/elastic/go-lumber v0.1.1 h1:aae5rSBnwBvdB0aShJ7AbOYPyvP1/wS/JIOC1A4D1DM=
 github.com/elastic/go-lumber v0.1.1/go.mod h1:DMVoFv7YM71enE9X5vWJWWv7wvQNtzXh7bPeKukDccY=
-github.com/elastic/go-sysinfo v1.14.2 h1:DeIy+pVfdRsd08Nx2Xjh+dUS+jrEEI7LGc29U/BKVWo=
-github.com/elastic/go-sysinfo v1.14.2/go.mod h1:jPSuTgXG+dhhh0GKIyI2Cso+w5lPJ5PvVqKlL8LV/Hk=
+github.com/elastic/go-sysinfo v1.15.0 h1:54pRFlAYUlVNQ2HbXzLVZlV+fxS7Eax49stzg95M4Xw=
+github.com/elastic/go-sysinfo v1.15.0/go.mod h1:jPSuTgXG+dhhh0GKIyI2Cso+w5lPJ5PvVqKlL8LV/Hk=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=

testing/benchmark/README.md

@@ -76,43 +76,34 @@ The main commands are:
 Helper commands
 
 - `~/.ssh/id_rsa_terraform`: Generates a new SSH key without passphrase for the worker VMs.
-- `terraform.tfvars`: Copies the examples tfvars and sets the `user_name` var with to `$USER`.
+- `terraform.tfvars`:
+    - Set `TFVARS_SOURCE` to use an explicit profile
+    - Otherwise the `terraform.tfvars.example` file is used
+    - Sets the `USER_NAME` var to `$USER`.
 - `apmbench`: Compiles the `apmbench` binary from the provided location (`APMBENCH_PATH`).
 
-### Override the docker image  and image tag
+### Override Docker Image
 
-Running `make docker-override-committed-version` will create new docker images for `kibana` and `elastic-agent`
-with local `apm` package and `apm-server` and a Terraform variable file. The file named 
-`docker_image.auto.tfvars` contains Terraform Docker image Terraform variables overrides. This file is not 
-overridden automatically, you need to remove it manually if present.
+There are two ways to override the docker images (or tags) that Terraform will provision:
 
-#### Override docker image tag
+1. **Manual**: To use an already existing container, change the values of the `docker_image_override` (or `docker_image_tag_override`) field in the profile defined by `TFVARS_SOURCE` (default is `terraform.tfvars`).
 
-It is possible to override the tag of the docker image that is run in the remote ESS deployment. You can
-specify any of the available tags (such as `8.3.0-SNAPSHOT` or a more specific tag `8.3.0-c655cda8-SNAPSHOT`).
-Alternatively, you can run `make docker-override-committed-version` in your shell, to have use the committed
-tags in the `docker-compose.yml` file in the repository root.
-
-#### Override the docker image
-
-It is also possible to override the docker image to one that is allowed to run in ESS. For more information
-on which repositories can be used, please refer to our internal docs. To override the docker image, you'll need
-to specify the full object of images that is defined in `variables.tf`: `docker_image_override`.
-Alternatively, you can run `make docker-override-committed-version` in your shell, to have use the committed
-tags in the `docker-compose.yml` file in the repository root.
+2. **Automatic**: To encode local changes in `apm-server`, run `make docker-override-committed-version`.
+    - This will build and push a new set of docker images to the internal docker registery.
+    - If `IMAGE_TAG` is not set, the committed tags will be used as defined in `docker-compose.yml` in the repository root.
+    - The file `docker_image.auto.tfvars` contains the Terraform variable overrides for the defined docker images. No need to override them manualy.
+    - This file is not overridden automatically, you need to remove it manually if present.
 
 ### Set APM index shards
 
-By default, the APM indices ship with `number_of_shards` set to `1`. To override this behavior, you can modify the
-`apm_shards` variable and individually set the setting for each of the component templates. See an example of how to
-do that in `terraform.tfvars.example`.
+By default, the APM indices ship with `number_of_shards` set to `1`. To override this behavior, you can modify the `apm_shards` variable and individually set the setting for each of the component templates. See an example of how to do that in `terraform.tfvars.example`.
 
 ### Delete all the APM data streams
 
-`make cleanup-elasticsearch` will delete all the APM data streams. This may be useful in case you'd like to re-run
-the benchmarks without destroying the deployment.
+`make cleanup-elasticsearch` will delete all the APM data streams. This may be useful in case you'd like to re-run the benchmarks without destroying the deployment.
 
 ### Slack reporting
+
 Reporting data is taken from the https://`<replace-with-kibana-benchmark-url>`/app/dashboards#/view/a5bc8390-2f8e-11ed-a369-052d8245fa04?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-30d,to:now))
 It's possible to add or modify any metric.
 

testing/infra/k8s/base/stack/apm-server.yaml

@@ -4,7 +4,7 @@ metadata:
   name: apm-server
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-263e2f29-SNAPSHOT
+  version: 9.0.0-8f0a2811-SNAPSHOT
   count: 1
   http:
     tls:

testing/infra/k8s/base/stack/elasticsearch.yaml

@@ -4,7 +4,7 @@ metadata:
   name: elasticsearch
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-263e2f29-SNAPSHOT
+  version: 9.0.0-8f0a2811-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin

testing/infra/k8s/base/stack/kibana.yaml

@@ -4,7 +4,7 @@ metadata:
   name: kibana
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-263e2f29-SNAPSHOT
+  version: 9.0.0-8f0a2811-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch

testing/infra/terraform/modules/tags/output.tf

@@ -2,7 +2,7 @@ locals {
   tags = {
     "division" : "engineering"
     "org" : "obs"
-    "team" : "apm-server"
+    "team" : "obs-ds-intake-services"
     "project" : var.project
     "build" : var.build
     "ephemeral" : "true"