Merge branch '8.x' into mergify/bp/8.x/pr-14256

.ci/scripts/push-pgo-pr.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eo pipefail
+
+PGO_BRANCH="update-pgo-$(date +%s)"
+cd $WORKSPACE_PATH
+git fetch origin main
+git checkout main
+git checkout -b $PGO_BRANCH
+mv $PROFILE_PATH x-pack/apm-server/default.pgo
+git add x-pack/apm-server/default.pgo
+git commit -m "PGO: Update default.pgo from benchmarks $WORKFLOW."
+git push -u origin $PGO_BRANCH
+gh pr create -B main -H $PGO_BRANCH -t "PGO: Update default.pgo" -b "Update default.pgo CPU profile from the benchmarks [workflow]($WORKFLOW)." -R elastic/apm-server
+gh pr merge --auto --delete-branch --squash $PGO_BRANCH

.github/workflows/add-to-docs-project.yml

@@ -11,6 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.label.name == 'Team:Docs'
     steps:
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "organization_projects": "write",
+              "issues": "read"
+            }
       - uses: octokit/[email protected]
         id: add_to_project
         with:
@@ -28,4 +39,4 @@ jobs:
           contentid: ${{ github.event.issue.node_id }}
         env:
           PROJECT_ID: "PVT_kwDOAGc3Zs0iZw"
-          GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}

.github/workflows/add-to-project.yml

@@ -14,7 +14,18 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "organization_projects": "write",
+              "issues": "read"
+            }
       - uses: actions/[email protected]
         with:
           project-url: https://github.com/orgs/elastic/projects/1286
-          github-token: ${{ secrets.APM_TECH_USER_TOKEN }}
+          github-token: ${{ steps.get_token.outputs.token }}

.github/workflows/benchmarks.yml

@@ -3,6 +3,11 @@ name: benchmarks
 on:
   workflow_dispatch:
     inputs:
+      runStandalone:
+        description: 'Run the benchmarks against standalone APM Server with Moxy'
+        required: false
+        type: boolean
+        default: false
       profile:
         description: 'The system profile used to run the benchmarks'
         required: false
@@ -21,10 +26,12 @@ on:
         required: false
         type: string
   schedule:
-    - cron: '0 17 * * *'
+    - cron: '0 17 * * *' # Scheduled regular benchmarks.
+    - cron: '0 5 */5 * *' # Scheduled PGO benchmarks.
 
 env:
   PNG_REPORT_FILE: out.png
+  BENCHMARK_CPU_OUT: default.pgo
   BENCHMARK_RESULT: benchmark-result.txt
   WORKING_DIRECTORY: testing/benchmark
 
@@ -38,12 +45,13 @@ jobs:
       run:
         working-directory: ${{ env.WORKING_DIRECTORY }}
     permissions:
-      contents: read
+      contents: write
       id-token: write
     env:
       SSH_KEY: ./id_rsa_terraform
       TF_VAR_private_key: ./id_rsa_terraform
       TF_VAR_public_key: ./id_rsa_terraform.pub
+      RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
       TF_VAR_BUILD_ID: ${{ github.run_id }}
       TF_VAR_ENVIRONMENT: ci
@@ -101,28 +109,48 @@ jobs:
           terraform_version: 1.3.7
           terraform_wrapper: false
 
+      - name: Init terraform module
+        id: init
+        run: make init
+
       - name: Build apmbench
         run: make apmbench $SSH_KEY terraform.tfvars
 
+      - name: Build APM Server and Moxy
+        if: ${{ env.RUN_STANDALONE == 'true' }}
+        run: |
+          make apm-server
+          make moxy
+
       - name: Override docker committed version
-        if: ${{ ! inputs.runOnStable }}
+        if: ${{ ! inputs.runOnStable && env.RUN_STANDALONE == 'false' }}
         run: make docker-override-committed-version
 
       - name: Spin up benchmark environment
         id: deploy
         run: |
-          make init apply
+          make apply
           admin_console_url=$(terraform output -raw admin_console_url)
           echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT"
           echo "-> infra setup done"
+        env:
+          TF_VAR_worker_region: ${{ env.AWS_REGION }}
+          TF_VAR_run_standalone: ${{ env.RUN_STANDALONE }}
 
       - name: Run benchmarks autotuned
         if: ${{ inputs.benchmarkAgents == '' }}
-        run: make run-benchmark-autotuned index-benchmark-results
+        run: make run-benchmark-autotuned
 
       - name: Run benchmarks self tuned
         if: ${{ inputs.benchmarkAgents != '' }}
-        run: make run-benchmark index-benchmark-results
+        run: make run-benchmark
+
+      - name: Cat standalone server logs
+        if: ${{ env.RUN_STANDALONE == 'true' && failure() }}
+        run: make cat-apm-server-logs
+
+      - name: Index benchmarks result
+        run: make index-benchmark-results
 
       - name: Download PNG
         run: >-
@@ -150,14 +178,11 @@ jobs:
 
       - name: Upload benchmark result
         uses: actions/upload-artifact@v4
-        if: always()
         with:
           name: benchmark-result
           path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_RESULT }}
           if-no-files-found: error
 
-<<<<<<< HEAD
-=======
       # The next section injects CPU profile collected by apmbench into the build.
       # By copying the profile, uploading it to the artifacts and pushing it
       # via a PR to update default.pgo.
@@ -209,10 +234,9 @@ jobs:
           GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
           WORKFLOW: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
 
->>>>>>> 3833927f8 (PGO: optimize collected profile file size (#14256))
       - name: Tear down benchmark environment
         if: always()
-        run: make destroy
+        run: make init destroy
 
       # Notify failure to Slack only on schedule (nightly run)
       - if: failure() && github.event_name == 'schedule'

.github/workflows/ci.yml

@@ -123,7 +123,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v45.0.2
+        uses: tj-actions/changed-files@c3a1bb2c992d77180ae65be6ae6c166cf40f857c # v45.0.3
         with:
           files: .go-version
 

.go-version

@@ -1 +1 @@
-1.23.1
+1.23.2

Makefile

@@ -176,7 +176,7 @@ docs/spec: go.mod
 # Beats synchronisation.
 ##############################################################################
 
-BEATS_VERSION?=8.x
+BEATS_VERSION?=main
 BEATS_MODULE:=github.com/elastic/beats/v7
 
 .PHONY: update-beats

NOTICE.txt

@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20240927142811-d130d84e0c8c
+Version: v7.0.0-alpha2.0.20240823193033-af33fad4d50b
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20240927142811-d130d84e0c8c/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20240823193033-af33fad4d50b/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -829,11 +829,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.12.0
+Version: v0.12.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -2379,12 +2379,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/gofrs/uuid/v5
-Version: v5.3.0
+Dependency : github.com/gofrs/uuid
+Version: v4.4.0+incompatible
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/[email protected]/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid@v4.4.0+incompatible/LICENSE:
 
 Copyright (C) 2013-2018 by Maxim Bublis <[email protected]>
 
@@ -6249,11 +6249,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
-Version: v0.29.0
+Version: v0.30.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.29.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.30.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 
@@ -6323,11 +6323,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
-Version: v0.24.0
+Version: v0.25.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.24.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.25.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 
@@ -6360,11 +6360,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/time
-Version: v0.6.0
+Version: v0.7.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/time@v0.6.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/time@v0.7.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 
@@ -6609,11 +6609,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/[email protected]/LIC
 
 --------------------------------------------------------------------------------
 Dependency : google.golang.org/protobuf
-Version: v1.34.2
+Version: v1.35.1
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.34.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.35.1/LICENSE:
 
 Copyright (c) 2018 The Go Authors. All rights reserved.
 
@@ -10597,6 +10597,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/gofrs/uuid/v5
+Version: v5.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/[email protected]/LICENSE:
+
+Copyright (C) 2013-2018 by Maxim Bublis <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/gogo/googleapis
 Version: v1.4.1
@@ -17067,11 +17097,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/crypto
-Version: v0.27.0
+Version: v0.28.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.27.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.28.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 
@@ -17141,11 +17171,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/sys
-Version: v0.25.0
+Version: v0.26.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.25.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.26.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 
@@ -17178,11 +17208,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/text
-Version: v0.18.0
+Version: v0.19.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.18.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.19.0/LICENSE:
 
 Copyright 2009 The Go Authors.
 

cmd/apm-server/main.go

@@ -25,7 +25,6 @@ import (
 )
 
 func main() {
-	beatcmd.InitBeatCmd()
 	rootCmd := beatcmd.NewRootCommand(beatcmd.BeatParams{
 		NewRunner: func(args beatcmd.RunnerParams) (beatcmd.Runner, error) {
 			return beater.NewRunner(beater.RunnerParams{