Merge branch 'main' into otlp-apis

.github/workflows/microbenchmark.yml

@@ -19,21 +19,14 @@ permissions:
 jobs:
   microbenchmark:
     runs-on: ubuntu-latest
-    # wait up to 1 hour
-    timeout-minutes: 60
+    timeout-minutes: 5
     steps:
-      - id: buildkite
-        name: Run buildkite pipeline
-        uses: elastic/apm-pipeline-library/.github/actions/buildkite@current
+      - name: Run buildkite pipeline
+        uses: elastic/oblt-actions/buildkite/[email protected]
         with:
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
+          token: ${{ secrets.BUILDKITE_TOKEN }}
           pipeline: apm-agent-microbenchmark
-          triggerMessage: "${{ github.repository }}@${{ github.ref_name }}"
-          waitFor: true
-          printBuildLogs: true
-          buildEnvVars: |
+          env-vars: |
             script=.ci/scripts/bench.sh
             repo=apm-server
             sha=${{ github.sha }}

NOTICE.txt

@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20240528110759-f411b0624c17
+Version: v7.0.0-alpha2.0.20240531144051-d77596ace183
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20240528110759-f411b0624c17/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20240531144051-d77596ace183/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -596,11 +596,11 @@ License Version 2.0.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.9.0
+Version: v7.10.0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.9.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.10.0/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 
@@ -829,11 +829,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.9.8
+Version: v0.9.10
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].10/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

dev_docs/RELEASES.md

@@ -9,34 +9,23 @@ For patch releases, only the version on the existing major and minor version bra
 
 ## Feature Freeze
 
-* For patch releases, ensure all relevant backport PRs are merged.
-  We use backport labels on PRs and automation to ensure labels are set.
+* For **patch releases**:
+  * ensure all relevant backport PRs are merged. We use backport labels on PRs and automation to ensure labels are set.
 
-* Update Changelog
-
-  * Review existing [changelogs/head](https://github.com/elastic/apm-server/tree/main/changelogs/head.asciidoc) to ensure all relevant notes have been added.
-  * Move changelog entries from _head_ to _release_version_:
-    * Minor version:
-      Create new changelog file from [changelogs/head.asciidoc](https://github.com/elastic/apm-server/blob/main/changelogs/head.asciidoc)
-      If changes should not be backported, keep them in the _changelogs/head.asciidoc_ file.
-      Don't forget to `include` and link to the new file in [main changelog](https://github.com/elastic/apm-server/blob/main/CHANGELOG.asciidoc) and the [release notes](https://github.com/elastic/apm-server/blob/main/docs/release-notes.asciidoc) file. [(Sample PR)](https://github.com/elastic/apm-server/pull/7956/files)
-    * Patch version: Add a new section to existing release notes. ([Sample PR](https://github.com/elastic/apm-server/pull/8313/files))
-  * Add `@elastic/obs-docs` as a reviewer.
 
 ## Day after Feature Freeze
 
-* For minor releases, cut a new release branch from `main` and update them.
-  * Release branch:
-    Update versions and ensure that the `BEATS_VERSION` in the Makefile is updated,
-    e.g. [#2803](https://github.com/elastic/apm-server/pull/2803/files).
-    Trigger a new beats update, once the beats branch is also created.
-    Remove the [changelogs/head.asciidoc](https://github.com/elastic/apm-server/blob/main/changelogs/head.asciidoc) file from the release branch.
-
-  * Main branch:
-    Update [.mergify.yml](https://github.com/elastic/apm-server/blob/main/.mergify.yml) with a new backport rule for the next version,
-    and update versions to next minor version, e.g. [#2804](https://github.com/elastic/apm-server/pull/2804).
-
-  The release manager will ping the teams, but you can already prepare this in advance on the day after Feature Freeze.
+* Trigger release workflow manually
+  * For **patch releases**: run the [`run-patch-release`](https://github.com/elastic/apm-server/actions/workflows/run-patch-release.yml) workflow.  
+    This workflow will: create the release branch; update version across codebase; commit and create PR targeting the release branch.
+    Release notes for patch releases must be manually added:
+    * Add a new section to the existing release notes file ([Sample PR](https://github.com/elastic/apm-server/pull/12680)).
+    * Review the [changelogs/head](https://github.com/elastic/apm-server/tree/main/changelogs/head.asciidoc) file and move relevant changelog entries from `head.asciidoc` to `release_version.asciidoc` if the change is backported to release_version. If changes do not apply to the version being released, keep them in the `head.asciidoc` file.
+    * Review the commits in the release to ensure all changes are reflected in the release notes.
+    * Add your PR to the documentation release issue ([Sample Issue](https://github.com/elastic/dev/issues/2485)).
+  * For **minor releases**: run the [`run-minor-release`](https://github.com/elastic/apm-server/actions/workflows/run-minor-release.yml) workflow.  
+    This workflow will: create the release branch; update the changelog for the release branch and open a PR targeting the release branch titled `<major>.<minor>: update docs`; create a PR on `main` titled `<major>.<minor>: update docs, mergify, versions and changelogs`. Before merging them compare commits between latest minor and the new minor versions and ensure all relevant PRs have been included in the Changelog. If not, amend it in both PRs. Request and wait a PR review from the team before merging.
+* The Release Manager will ping the team to align the release process
 
 * Update dependencies
 
@@ -55,11 +44,13 @@ For patch releases, only the version on the existing major and minor version bra
 
 * Test plan
 
-  Create a github issue for testing the release branch (follow the GitHub issue template for the test plan), It should contain:
-  * A link to all PRs in the APM Server repository that need to be tested manually. Use the `test-plan*` labels and the version labels
-    to create an overview over the PRs that need testing. For example, [test plan link for 8.3.0](https://github.com/elastic/apm-server/issues?q=label%3Atest-plan+is%3Aclosed+label%3Av8.3.0).
-  * Add other test cases that require manual testing, such as test scenarios on ESS, that are not covered by automated tests or
-    OS compatibility smoke tests for supporting new operating systems.
+  Create a github issue for testing the release branch ([use the GitHub issue `test plan` template](https://github.com/elastic/apm-server/issues/new?assignees=&labels=test-plan&projects=&template=test-plan.md)), It should contain:
+  * A link to all PRs in the APM Server repository that need to be tested manually to create an overview over the PRs that need testing.
+    Use the `test-plan` label and the version label (create it if it does not exist). For example, [this was 8.13.0 test plan](https://github.com/elastic/apm-server/issues/12822)
+    and here you can find [all previous test plans](https://github.com/elastic/apm-server/issues?q=label%3Atest-plan+is%3Aclosed).
+    What we aim for is testing all functional changes applied to the new version. Review any PR updating `elastic/go-docappender` and `elastic/apm-data` dependencies, as some functional changes happens through these dependencies.  
+   Any non-functional change or any change that is already covered by automated tests must not be included.
+  * Add other test cases that require manual testing, such as test scenarios on ESS, that are not covered by automated tests or OS compatibility smoke tests for supporting new operating systems.
 
 ## Between feature freeze and release
 

docker-compose.yml

@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-1b4d655a-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-b3ff27eb-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.15.0-1b4d655a-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.15.0-b3ff27eb-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.15.0-1b4d655a-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.15.0-b3ff27eb-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

go.mod

@@ -10,9 +10,9 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6
 	github.com/elastic/apm-data v1.1.0
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240528110759-f411b0624c17
-	github.com/elastic/elastic-agent-client/v7 v7.9.0
-	github.com/elastic/elastic-agent-libs v0.9.8
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240531144051-d77596ace183
+	github.com/elastic/elastic-agent-client/v7 v7.10.0
+	github.com/elastic/elastic-agent-libs v0.9.10
 	github.com/elastic/elastic-agent-system-metrics v0.10.2
 	github.com/elastic/gmux v0.3.2
 	github.com/elastic/go-docappender/v2 v2.1.2

go.sum

@@ -147,14 +147,14 @@ github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HE
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo=
 github.com/elastic/apm-data v1.1.0 h1:5ahaTQwWb6+OmrXrc1dzfD2wjvTG7yJRaldcsc9dIc4=
 github.com/elastic/apm-data v1.1.0/go.mod h1:TLHPd2H8wYlf3FIqriQ018NZzE1FLC6VtsQ6DN0vRmY=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240528110759-f411b0624c17 h1:yP3ds+Or83h9tb2GnokmVX1NaVrN//V3iZVYWQAUBLA=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240528110759-f411b0624c17/go.mod h1:HjUJr+gw2jebZYFMKc9TNbgwjHsNSo6HUSonhC0u62U=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240531144051-d77596ace183 h1:TZ0bnwJTCAMtEayuGvp15S+zBaaDQrZtUcfTJXfNKTk=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240531144051-d77596ace183/go.mod h1:Sgu+4L2XSFMQ0YvZLxbhKxkesTQkoetgeUhdkel9uIM=
 github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP+WioV73ia17Et+UlFbz8=
 github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8=
-github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE=
-github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
-github.com/elastic/elastic-agent-libs v0.9.8 h1:fwl3hp0gNmKkuERcUQTwe4cyIK6M0jJkv16EIsB6Apw=
-github.com/elastic/elastic-agent-libs v0.9.8/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc=
+github.com/elastic/elastic-agent-client/v7 v7.10.0 h1:qcz5EHOI+Jh8QHVGLAOQ9BRXORTYjcziXq1y4reESAk=
+github.com/elastic/elastic-agent-client/v7 v7.10.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
+github.com/elastic/elastic-agent-libs v0.9.10 h1:t6eM2s2EXv/AzXXdFy8eWS+/HAzE7AZL1yo3lSAPsfo=
+github.com/elastic/elastic-agent-libs v0.9.10/go.mod h1:CpBqDK8Aql/ou57UR4bWNQt0Cfr14rTRYBmWyiwDpW0=
 github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc=
 github.com/elastic/elastic-agent-system-metrics v0.10.2/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI=
 github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0=
@@ -780,6 +780,8 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk=
+gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=

internal/beater/beater.go

@@ -779,6 +779,9 @@ func docappenderConfig(
 		"docappender.DocumentBufferSize", opts.DocumentBufferSize, memLimit,
 	)
 	if opts.MaxRequests > 0 {
+		logger.Infof("docappender.MaxRequests set to %d based on config value",
+			opts.MaxRequests,
+		)
 		return opts
 	}
 	// This formula yields the following max requests for APM Server sized:

testing/infra/k8s/base/stack/apm-server.yaml

@@ -3,7 +3,7 @@ kind: ApmServer
 metadata:
   name: apm-server
 spec:
-  version: 8.15.0-1b4d655a-SNAPSHOT
+  version: 8.15.0-b3ff27eb-SNAPSHOT
   count: 1
   http:
     tls:

testing/infra/k8s/base/stack/elasticsearch.yaml

@@ -3,7 +3,7 @@ kind: Elasticsearch
 metadata:
   name: elasticsearch
 spec:
-  version: 8.15.0-1b4d655a-SNAPSHOT
+  version: 8.15.0-b3ff27eb-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin

testing/infra/k8s/base/stack/kibana.yaml

@@ -3,7 +3,7 @@ kind: Kibana
 metadata:
   name: kibana
 spec:
-  version: 8.15.0-1b4d655a-SNAPSHOT
+  version: 8.15.0-b3ff27eb-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch