diff --git a/.ci/scripts/push-pgo-pr.sh b/.ci/scripts/push-pgo-pr.sh new file mode 100755 index 00000000000..93366424c7e --- /dev/null +++ b/.ci/scripts/push-pgo-pr.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -eo pipefail + +PGO_BRANCH="update-pgo-$(date +%s)" +cd $WORKSPACE_PATH +git fetch origin main +git checkout main +git checkout -b $PGO_BRANCH +mv $PROFILE_PATH x-pack/apm-server/default.pgo +git add x-pack/apm-server/default.pgo +git commit -m "PGO: Update default.pgo from benchmarks $WORKFLOW." +git push -u origin $PGO_BRANCH +gh pr create -B main -H $PGO_BRANCH -t "PGO: Update default.pgo" -b "Update default.pgo CPU profile from the benchmarks [workflow]($WORKFLOW)." -R elastic/apm-server +gh pr merge --auto --delete-branch --squash $PGO_BRANCH \ No newline at end of file diff --git a/.github/workflows/add-to-docs-project.yml b/.github/workflows/add-to-docs-project.yml index 85125dc1bc3..57de59076a1 100644 --- a/.github/workflows/add-to-docs-project.yml +++ b/.github/workflows/add-to-docs-project.yml @@ -11,6 +11,17 @@ jobs: runs-on: ubuntu-latest if: github.event.label.name == 'Team:Docs' steps: + - name: Get token + id: get_token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + with: + app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }} + private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }} + permissions: >- + { + "organization_projects": "write", + "issues": "read" + } - uses: octokit/graphql-action@v2.x id: add_to_project with: @@ -28,4 +39,4 @@ jobs: contentid: ${{ github.event.issue.node_id }} env: PROJECT_ID: "PVT_kwDOAGc3Zs0iZw" - GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }} + GITHUB_TOKEN: ${{ steps.get_token.outputs.token }} diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index fcd299bd85e..c89595bd3a0 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -14,7 +14,18 @@ jobs: name: Add issue to project runs-on: ubuntu-latest steps: + - name: Get token + id: get_token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + with: + app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }} + private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }} + permissions: >- + { + "organization_projects": "write", + "issues": "read" + } - uses: actions/add-to-project@v1.0.2 with: project-url: https://github.com/orgs/elastic/projects/1286 - github-token: ${{ secrets.APM_TECH_USER_TOKEN }} \ No newline at end of file + github-token: ${{ steps.get_token.outputs.token }} diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index eda117cdc7a..799cbcd1800 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -3,6 +3,11 @@ name: benchmarks on: workflow_dispatch: inputs: + runStandalone: + description: 'Run the benchmarks against standalone APM Server with Moxy' + required: false + type: boolean + default: false profile: description: 'The system profile used to run the benchmarks' required: false @@ -21,10 +26,12 @@ on: required: false type: string schedule: - - cron: '0 17 * * *' + - cron: '0 17 * * *' # Scheduled regular benchmarks. + - cron: '0 5 */5 * *' # Scheduled PGO benchmarks. env: PNG_REPORT_FILE: out.png + BENCHMARK_CPU_OUT: default.pgo BENCHMARK_RESULT: benchmark-result.txt WORKING_DIRECTORY: testing/benchmark @@ -38,12 +45,13 @@ jobs: run: working-directory: ${{ env.WORKING_DIRECTORY }} permissions: - contents: read + contents: write id-token: write env: SSH_KEY: ./id_rsa_terraform TF_VAR_private_key: ./id_rsa_terraform TF_VAR_public_key: ./id_rsa_terraform.pub + RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }} TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile TF_VAR_BUILD_ID: ${{ github.run_id }} TF_VAR_ENVIRONMENT: ci @@ -101,28 +109,48 @@ jobs: terraform_version: 1.3.7 terraform_wrapper: false + - name: Init terraform module + id: init + run: make init + - name: Build apmbench run: make apmbench $SSH_KEY terraform.tfvars + - name: Build APM Server and Moxy + if: ${{ env.RUN_STANDALONE == 'true' }} + run: | + make apm-server + make moxy + - name: Override docker committed version - if: ${{ ! inputs.runOnStable }} + if: ${{ ! inputs.runOnStable && env.RUN_STANDALONE == 'false' }} run: make docker-override-committed-version - name: Spin up benchmark environment id: deploy run: | - make init apply + make apply admin_console_url=$(terraform output -raw admin_console_url) echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT" echo "-> infra setup done" + env: + TF_VAR_worker_region: ${{ env.AWS_REGION }} + TF_VAR_run_standalone: ${{ env.RUN_STANDALONE }} - name: Run benchmarks autotuned if: ${{ inputs.benchmarkAgents == '' }} - run: make run-benchmark-autotuned index-benchmark-results + run: make run-benchmark-autotuned - name: Run benchmarks self tuned if: ${{ inputs.benchmarkAgents != '' }} - run: make run-benchmark index-benchmark-results + run: make run-benchmark + + - name: Cat standalone server logs + if: ${{ env.RUN_STANDALONE == 'true' && failure() }} + run: make cat-apm-server-logs + + - name: Index benchmarks result + run: make index-benchmark-results - name: Download PNG run: >- @@ -150,15 +178,65 @@ jobs: - name: Upload benchmark result uses: actions/upload-artifact@v4 - if: always() with: name: benchmark-result path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_RESULT }} if-no-files-found: error + # The next section injects CPU profile collected by apmbench into the build. + # By copying the profile, uploading it to the artifacts and pushing it + # via a PR to update default.pgo. + + - name: Copy CPU profile + run: make cp-cpuprof + + - name: Upload CPU profile + uses: actions/upload-artifact@v4 + with: + name: cpu-profile + path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_CPU_OUT }} + if-no-files-found: error + + - name: Get token + id: get_token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + with: + app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }} + private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }} + permissions: >- + { + "contents": "write", + "pull_requests": "write" + } + + # Required to use a service account, otherwise PRs created by + # GitHub bot won't trigger any CI builds. + # See https://github.com/peter-evans/create-pull-request/issues/48#issuecomment-537478081 + - name: Configure git user + uses: elastic/oblt-actions/git/setup@v1 + with: + github-token: ${{ steps.get_token.outputs.token }} + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6.1.0 + with: + gpg_private_key: ${{ secrets.APM_SERVER_RELEASE_GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.APM_SERVER_RELEASE_PASSPHRASE }} + git_user_signingkey: true + git_commit_gpgsign: true + + - name: Open PGO PR + if: ${{ env.RUN_STANDALONE == 'true' }} + run: ${{ github.workspace }}/.ci/scripts/push-pgo-pr.sh + env: + WORKSPACE_PATH: ${{ github.workspace }} + PROFILE_PATH: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_CPU_OUT }} + GITHUB_TOKEN: ${{ steps.get_token.outputs.token }} + WORKFLOW: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + - name: Tear down benchmark environment if: always() - run: make destroy + run: make init destroy # Notify failure to Slack only on schedule (nightly run) - if: failure() && github.event_name == 'schedule' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c7fdd641e6..4b5914a6b44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,7 @@ jobs: - uses: actions/checkout@v4 - name: Get changed files id: changed-files - uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v45.0.2 + uses: tj-actions/changed-files@c3a1bb2c992d77180ae65be6ae6c166cf40f857c # v45.0.3 with: files: .go-version diff --git a/.go-version b/.go-version index 49e0a31d496..14bee92c9e7 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.23.1 +1.23.2 diff --git a/Makefile b/Makefile index fc7e4091bf9..30e5d691ff4 100644 --- a/Makefile +++ b/Makefile @@ -176,7 +176,7 @@ docs/spec: go.mod # Beats synchronisation. ############################################################################## -BEATS_VERSION?=8.x +BEATS_VERSION?=main BEATS_MODULE:=github.com/elastic/beats/v7 .PHONY: update-beats diff --git a/NOTICE.txt b/NOTICE.txt index eae8f345b6b..751511b93bf 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.13. -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20240927142811-d130d84e0c8c +Version: v7.0.0-alpha2.0.20240823193033-af33fad4d50b Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20240927142811-d130d84e0c8c/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20240823193033-af33fad4d50b/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -829,11 +829,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.12.0 +Version: v0.12.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE: Apache License Version 2.0, January 2004 @@ -2379,12 +2379,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/gofrs/uuid/v5 -Version: v5.3.0 +Dependency : github.com/gofrs/uuid +Version: v4.4.0+incompatible Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/v5@v5.3.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid@v4.4.0+incompatible/LICENSE: Copyright (C) 2013-2018 by Maxim Bublis @@ -6609,11 +6609,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.67.1/LIC -------------------------------------------------------------------------------- Dependency : google.golang.org/protobuf -Version: v1.34.2 +Version: v1.35.1 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.34.2/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.35.1/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. @@ -10597,6 +10597,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/gofrs/uuid/v5 +Version: v5.2.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/v5@v5.2.0/LICENSE: + +Copyright (C) 2013-2018 by Maxim Bublis + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/gogo/googleapis Version: v1.4.1 diff --git a/cmd/apm-server/main.go b/cmd/apm-server/main.go index 30a4789c57e..72244c3d3d3 100644 --- a/cmd/apm-server/main.go +++ b/cmd/apm-server/main.go @@ -25,7 +25,6 @@ import ( ) func main() { - beatcmd.InitBeatCmd() rootCmd := beatcmd.NewRootCommand(beatcmd.BeatParams{ NewRunner: func(args beatcmd.RunnerParams) (beatcmd.Runner, error) { return beater.NewRunner(beater.RunnerParams{ diff --git a/docker-compose.yml b/docker-compose.yml index 8a7bf1c89a8..70a5dbfb7ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-0a160e40-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-644441e6-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.16.0-0a160e40-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.16.0-644441e6-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.16.0-0a160e40-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.16.0-644441e6-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/go.mod b/go.mod index 2d3ccf1ecf6..fd9043f0fae 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v1.1.0 github.com/elastic/apm-data v1.13.1 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240927142811-d130d84e0c8c + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b github.com/elastic/elastic-agent-client/v7 v7.16.0 - github.com/elastic/elastic-agent-libs v0.12.0 + github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.3 github.com/elastic/gmux v0.3.2 github.com/elastic/go-docappender/v2 v2.3.0 @@ -19,7 +19,7 @@ require ( github.com/elastic/go-ucfg v0.8.8 github.com/go-sourcemap/sourcemap v2.1.4+incompatible github.com/gofrs/flock v0.12.1 - github.com/gofrs/uuid/v5 v5.3.0 + github.com/gofrs/uuid v4.4.0+incompatible github.com/gogo/protobuf v1.3.2 github.com/google/go-cmp v0.6.0 github.com/gorilla/mux v1.8.1 @@ -53,7 +53,7 @@ require ( golang.org/x/term v0.25.0 golang.org/x/time v0.7.0 google.golang.org/grpc v1.67.1 - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -98,6 +98,7 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gofrs/uuid/v5 v5.2.0 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/golang/glog v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index b52f2aa3309..a714728b1be 100644 --- a/go.sum +++ b/go.sum @@ -133,14 +133,14 @@ github.com/elastic/apm-aggregation v1.1.0 h1:eTHGd5w99JoRFJ763MJP6nUX4hkkeSil3KW github.com/elastic/apm-aggregation v1.1.0/go.mod h1:YBQ77Jt7vOy2/7w4q5SsEifhc0V02dJxRcyyeC6HsvQ= github.com/elastic/apm-data v1.13.1 h1:vQUWqi9H6QC4FCZ05kILkE+2BeKWw6iT+NXXyadWJm8= github.com/elastic/apm-data v1.13.1/go.mod h1:m2FSH7alN07VmtmN10KmgxAa8tgXlJDe/rC0g4FnZUI= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240927142811-d130d84e0c8c h1:lGnbSjxi+rTsyN8qlJR/9FAmlk6oS0RIaqrzYDMHaOQ= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240927142811-d130d84e0c8c/go.mod h1:tBsm3eI27+LJqaABPmGLgr/6RcQ50lHsQxbsfDGlHHQ= -github.com/elastic/elastic-agent-autodiscover v0.8.2 h1:Fs2FhR33AMBPfm5/jz4drVzaEZaqOIHlDBvGtkUZdIk= -github.com/elastic/elastic-agent-autodiscover v0.8.2/go.mod h1:VZnU53EVaFTxR8Xf6YsLN8FHD5DKQzHSPlKax9/4w+o= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b h1:i3WdOw//Vg76Vseqxgn47GPTVqUSucAdE42jkih1ViM= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b/go.mod h1:m8d3EBr2z7YvWfxRgoE7SsUNbMTpyWXi67fG7B7vn7w= +github.com/elastic/elastic-agent-autodiscover v0.8.1 h1:u6TWqh7wfevu6S4GUq4SIxYBRo4b/P5RZmx/rSvT10A= +github.com/elastic/elastic-agent-autodiscover v0.8.1/go.mod h1:0gzGsaDCAqBfUZjuCqqWsSI60eaZ778A5tQZV72rPV0= github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0= github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.12.0 h1:xfVVCYIaI6XEPVpJNq7HQav7O/VxLj+YbQK/poWb7wA= -github.com/elastic/elastic-agent-libs v0.12.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= +github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0= +github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A= github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= @@ -200,8 +200,10 @@ github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TC github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= -github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= -github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM= +github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -583,8 +585,8 @@ google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/grpc/examples v0.0.0-20231016154744-cb430bed4d27 h1:EB/3dtnYKOItaNPpOI/HmOCGbVZUiXcstRfiuxN+cFg= google.golang.org/grpc/examples v0.0.0-20231016154744-cb430bed4d27/go.mod h1:Crtq1t+mykyL5d6PR3z8zCxKx/Qjq/mlPWDPoWJANYA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/beatcmd/beat.go b/internal/beatcmd/beat.go index 8a8b38d0f79..8a62e1c8982 100644 --- a/internal/beatcmd/beat.go +++ b/internal/beatcmd/beat.go @@ -31,7 +31,7 @@ import ( "strings" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -123,7 +123,6 @@ func NewBeat(args BeatParams) (*Beat, error) { Keystore: keystore, Config: &beat.BeatConfig{Output: cfg.Output}, BeatConfig: cfg.APMServer, - Registry: reload.NewRegistry(), }, Config: cfg, newRunner: args.NewRunner, @@ -153,7 +152,7 @@ func (b *Beat) init() error { logp.Info("Beat ID: %v", b.Info.ID) // Initialize central config manager. - manager, err := management.NewManager(b.Config.Management, b.Registry) + manager, err := management.NewManager(b.Config.Management, reload.RegisterV2) if err != nil { return err } @@ -367,7 +366,7 @@ func (b *Beat) Run(ctx context.Context) error { } if b.Manager.Enabled() { - reloader, err := NewReloader(b.Info, b.Registry, b.newRunner) + reloader, err := NewReloader(b.Info, b.newRunner) if err != nil { return err } diff --git a/internal/beatcmd/beat_test.go b/internal/beatcmd/beat_test.go index ca1da3e075a..2599647fc21 100644 --- a/internal/beatcmd/beat_test.go +++ b/internal/beatcmd/beat_test.go @@ -125,6 +125,10 @@ func TestUnmanagedOutputRequired(t *testing.T) { } func TestRunManager(t *testing.T) { + oldRegistry := reload.RegisterV2 + defer func() { reload.RegisterV2 = oldRegistry }() + reload.RegisterV2 = reload.NewRegistry() + // Register our own mock management implementation. manager := newMockManager() management.SetManagerFactory(func(cfg *config.C, registry *reload.Registry) (management.Manager, error) { @@ -147,20 +151,20 @@ func TestRunManager(t *testing.T) { expectEvent(t, manager.started, "manager should have been started") expectNoEvent(t, manager.stopped, "manager should not have been stopped") - err := b.Registry.GetInputList().Reload([]*reload.ConfigWithMeta{{ + err := reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{ Config: config.MustNewConfigFrom(`{ "revision": 1, "apm-server.host": "localhost:1234" }`), }}) assert.NoError(t, err) - err = b.Registry.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ + err = reload.RegisterV2.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{"console.enabled": true}`), }) assert.NoError(t, err) expectRunnerParams(t, calls) - err = b.Registry.GetReloadableAPM().Reload(&reload.ConfigWithMeta{ + err = reload.RegisterV2.GetReloadableAPM().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{"elastic.enabled": true, "elastic.environment": "testenv"}`), }) assert.NoError(t, err) @@ -232,6 +236,9 @@ func resetGlobals() { registry.Clear() } } + + // Create a new reload registry, as the Beat.Run method will register with it. + reload.RegisterV2 = reload.NewRegistry() } type runnerFunc func(ctx context.Context) error diff --git a/internal/beatcmd/init.go b/internal/beatcmd/init.go index 2ccecda29f5..bbf657006ac 100644 --- a/internal/beatcmd/init.go +++ b/internal/beatcmd/init.go @@ -19,7 +19,6 @@ package beatcmd import ( cryptorand "crypto/rand" - "flag" "log" "math" "math/big" @@ -39,7 +38,7 @@ import ( _ "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue" ) -func InitBeatCmd() { +func init() { initRand() initFlags() } @@ -63,7 +62,6 @@ func initFlags() { } } - flag.Parse() if err := cfgfile.HandleFlags(); err != nil { log.Fatal(err) } diff --git a/internal/beatcmd/reloader.go b/internal/beatcmd/reloader.go index f3dad9ba672..297fa328cc9 100644 --- a/internal/beatcmd/reloader.go +++ b/internal/beatcmd/reloader.go @@ -58,20 +58,20 @@ type Runner interface { // NewReloader returns a new Reloader which creates Runners using the provided // beat.Info and NewRunnerFunc. -func NewReloader(info beat.Info, registry *reload.Registry, newRunner NewRunnerFunc) (*Reloader, error) { +func NewReloader(info beat.Info, newRunner NewRunnerFunc) (*Reloader, error) { r := &Reloader{ info: info, logger: logp.NewLogger(""), newRunner: newRunner, stopped: make(chan struct{}), } - if err := registry.RegisterList(reload.InputRegName, reloadableListFunc(r.reloadInputs)); err != nil { + if err := reload.RegisterV2.RegisterList(reload.InputRegName, reloadableListFunc(r.reloadInputs)); err != nil { return nil, fmt.Errorf("failed to register inputs reloader: %w", err) } - if err := registry.Register(reload.OutputRegName, reload.ReloadableFunc(r.reloadOutput)); err != nil { + if err := reload.RegisterV2.Register(reload.OutputRegName, reload.ReloadableFunc(r.reloadOutput)); err != nil { return nil, fmt.Errorf("failed to register output reloader: %w", err) } - if err := registry.Register(reload.APMRegName, reload.ReloadableFunc(r.reloadAPMTracing)); err != nil { + if err := reload.RegisterV2.Register(reload.APMRegName, reload.ReloadableFunc(r.reloadAPMTracing)); err != nil { return nil, fmt.Errorf("failed to register apm tracing reloader: %w", err) } return r, nil diff --git a/internal/beatcmd/reloader_test.go b/internal/beatcmd/reloader_test.go index 540838bbe29..83a7fb38b6e 100644 --- a/internal/beatcmd/reloader_test.go +++ b/internal/beatcmd/reloader_test.go @@ -33,6 +33,10 @@ import ( ) func TestReloader(t *testing.T) { + oldRegistry := reload.RegisterV2 + defer func() { reload.RegisterV2 = oldRegistry }() + reload.RegisterV2 = reload.NewRegistry() + type runner struct { running chan struct{} stopped chan struct{} @@ -55,9 +59,7 @@ func TestReloader(t *testing.T) { panic("unreachable") } - registry := reload.NewRegistry() - - reloader, err := NewReloader(beat.Info{}, registry, func(args RunnerParams) (Runner, error) { + reloader, err := NewReloader(beat.Info{}, func(args RunnerParams) (Runner, error) { if shouldError, _ := args.Config.Bool("error", -1); shouldError { return nil, errors.New("no runner for you") } @@ -84,29 +86,29 @@ func TestReloader(t *testing.T) { // No reload until there's input, output, apm tracing configuration. assertNoReload() - err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{ + err = reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{ Config: config.MustNewConfigFrom(`{}`), }}) assert.EqualError(t, err, "failed to extract input config revision: missing field accessing 'revision'") assertNoReload() - err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{ + err = reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{ Config: config.MustNewConfigFrom(`{"revision": 1}`), }}) assert.NoError(t, err) assertNoReload() - err = registry.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ + err = reload.RegisterV2.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{}`), }) assert.NoError(t, err) assertNoReload() // an output must be set - err = registry.GetReloadableAPM().Reload(nil) + err = reload.RegisterV2.GetReloadableAPM().Reload(nil) assert.NoError(t, err) assertNoReload() - err = registry.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ + err = reload.RegisterV2.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{"console.enabled": true}`), }) assert.NoError(t, err) @@ -116,14 +118,14 @@ func TestReloader(t *testing.T) { expectEvent(t, r1.running, "runner should have been started") expectNoEvent(t, r1.stopped, "runner should not have been stopped") - err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{ + err = reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{ Config: config.MustNewConfigFrom(`{"revision": 2, "error": true}`), }}) assert.EqualError(t, err, "failed to load input config: no runner for you") assertNoReload() // error occurred during reload, nothing changes expectNoEvent(t, r1.stopped, "runner should not have been stopped") - err = registry.GetInputList().Reload([]*reload.ConfigWithMeta{{ + err = reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{ Config: config.MustNewConfigFrom(`{"revision": 3}`), }}) assert.NoError(t, err) @@ -132,7 +134,7 @@ func TestReloader(t *testing.T) { expectEvent(t, r2.running, "new runner should have been started") expectNoEvent(t, r2.stopped, "new runner should not have been stopped") - err = registry.GetReloadableAPM().Reload(&reload.ConfigWithMeta{ + err = reload.RegisterV2.GetReloadableAPM().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{"elastic.enabled": true, "elastic.api_key": "boo"}`), }) assert.NoError(t, err) @@ -146,11 +148,13 @@ func TestReloader(t *testing.T) { } func TestReloaderNewRunnerParams(t *testing.T) { - registry := reload.NewRegistry() + oldRegistry := reload.RegisterV2 + defer func() { reload.RegisterV2 = oldRegistry }() + reload.RegisterV2 = reload.NewRegistry() calls := make(chan RunnerParams, 1) info := beat.Info{Beat: "not-apm-server", Version: "0.0.1"} - reloader, err := NewReloader(info, registry, func(args RunnerParams) (Runner, error) { + reloader, err := NewReloader(info, func(args RunnerParams) (Runner, error) { calls <- args return runnerFunc(func(ctx context.Context) error { <-ctx.Done() @@ -165,18 +169,18 @@ func TestReloaderNewRunnerParams(t *testing.T) { defer func() { assert.NoError(t, g.Wait()) }() defer cancel() - registry.GetInputList().Reload([]*reload.ConfigWithMeta{{ + reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{ Config: config.MustNewConfigFrom(`{"revision": 1, "input": 123}`), }}) // reloader will wait until input and output are available. // triggering APM reload before output reload will let the params to contain // the apm tracing config too in this test setup - registry.GetReloadableAPM().Reload(&reload.ConfigWithMeta{ + reload.RegisterV2.GetReloadableAPM().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{"elastic.environment": "test"}`), }) - registry.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ + reload.RegisterV2.GetReloadableOutput().Reload(&reload.ConfigWithMeta{ Config: config.MustNewConfigFrom(`{"console.enabled": true}`), }) args := <-calls diff --git a/internal/beater/middleware/log_middleware.go b/internal/beater/middleware/log_middleware.go index 1198fa83271..b16cb7092c3 100644 --- a/internal/beater/middleware/log_middleware.go +++ b/internal/beater/middleware/log_middleware.go @@ -20,7 +20,7 @@ package middleware import ( "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "go.elastic.co/apm/v2" diff --git a/packaging/docker/Dockerfile.wolfi b/packaging/docker/Dockerfile.wolfi index 6d59d3f6b56..b4990957aec 100644 --- a/packaging/docker/Dockerfile.wolfi +++ b/packaging/docker/Dockerfile.wolfi @@ -30,7 +30,7 @@ RUN sed -i 's/localhost:9200/elasticsearch:9200/' apm-server.yml # Build stage 1 # Copy prepared files from the previous stage and complete the image. ################################################################################ -FROM cgr.dev/chainguard/static:latest@sha256:d2a76860057c1260ea5dc8ae4e18beff5ccfb1b67004295c9ab6951833e93de7 +FROM cgr.dev/chainguard/static:latest@sha256:f0444e0c32aa0d22cca6de79e160306e9f24140fea0ac97c41789ee59814aa87 ARG TARGETARCH ARG BUILD_DATE ARG VERSION diff --git a/systemtest/approvals/TestIntake/Errors.approved.json b/systemtest/approvals/TestIntake/Errors.approved.json index f34d403c731..3e1e159693e 100644 --- a/systemtest/approvals/TestIntake/Errors.approved.json +++ b/systemtest/approvals/TestIntake/Errors.approved.json @@ -169,18 +169,10 @@ "bar", "baz" ], - "Content-Type": [ - "text/html" - ], - "Cookie": [ - "c1=v1,c2=v2" - ], - "Some-Other-Header": [ - "foo" - ], - "User-Agent": [ - "Mozilla Chrome Edge" - ] + "Content-Type": "text/html", + "Cookie": "c1=v1,c2=v2", + "Some-Other-Header": "foo", + "User-Agent": "Mozilla Chrome Edge" } ], "http.request.method": [ @@ -194,9 +186,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ diff --git a/systemtest/approvals/TestIntake/ErrorsTxID.approved.json b/systemtest/approvals/TestIntake/ErrorsTxID.approved.json index dc21fa8e33a..79c6cd3834d 100644 --- a/systemtest/approvals/TestIntake/ErrorsTxID.approved.json +++ b/systemtest/approvals/TestIntake/ErrorsTxID.approved.json @@ -104,22 +104,14 @@ ], "http.request.headers": [ { - "Content-Length": [ - "0" - ], + "Content-Length": "0", "Cookie": [ "c1=v1", "c2=v2" ], - "Elastic-Apm-Traceparent": [ - "00-8c21b4b556467a0b17ae5da959b5f388-31301f1fb2998121-01" - ], - "Forwarded": [ - "for=192.168.0.1" - ], - "Host": [ - "opbeans-java:3000" - ] + "Elastic-Apm-Traceparent": "00-8c21b4b556467a0b17ae5da959b5f388-31301f1fb2998121-01", + "Forwarded": "for=192.168.0.1", + "Host": "opbeans-java:3000" } ], "http.request.method": [ @@ -130,9 +122,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ diff --git a/systemtest/approvals/TestIntake/Events.approved.json b/systemtest/approvals/TestIntake/Events.approved.json index e1d4848c822..9ccf5a3a4be 100644 --- a/systemtest/approvals/TestIntake/Events.approved.json +++ b/systemtest/approvals/TestIntake/Events.approved.json @@ -107,22 +107,14 @@ ], "http.request.headers": [ { - "Content-Length": [ - "0" - ], + "Content-Length": "0", "Cookie": [ "c1=v1", "c2=v2" ], - "Elastic-Apm-Traceparent": [ - "00-8c21b4b556467a0b17ae5da959b5f388-31301f1fb2998121-01" - ], - "Forwarded": [ - "for=192.168.0.1" - ], - "Host": [ - "opbeans-java:3000" - ] + "Elastic-Apm-Traceparent": "00-8c21b4b556467a0b17ae5da959b5f388-31301f1fb2998121-01", + "Forwarded": "for=192.168.0.1", + "Host": "opbeans-java:3000" } ], "http.request.method": [ @@ -133,9 +125,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ @@ -567,9 +557,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.status_code": [ @@ -816,15 +804,9 @@ ], "http.request.headers": [ { - "Content-Type": [ - "text/html" - ], - "Cookie": [ - "c1=v1,c2=v2" - ], - "Elastic-Apm-Traceparent": [ - "00-33a0bd4cceff0370a7c57d807032688e-69feaabc5b88d7e8-01" - ], + "Content-Type": "text/html", + "Cookie": "c1=v1,c2=v2", + "Elastic-Apm-Traceparent": "00-33a0bd4cceff0370a7c57d807032688e-69feaabc5b88d7e8-01", "User-Agent": [ "Mozilla/5.0(Macintosh;IntelMacOSX10_10_5)AppleWebKit/537.36(KHTML,likeGecko)Chrome/51.0.2704.103Safari/537.36", "MozillaChromeEdge" @@ -845,9 +827,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ diff --git a/systemtest/approvals/TestIntake/Transactions.approved.json b/systemtest/approvals/TestIntake/Transactions.approved.json index d7603216333..65368d51fe4 100644 --- a/systemtest/approvals/TestIntake/Transactions.approved.json +++ b/systemtest/approvals/TestIntake/Transactions.approved.json @@ -640,15 +640,9 @@ "bar", "baz" ], - "Content-Type": [ - "text/html" - ], - "Cookie": [ - "c1=v1, c2=v2" - ], - "Some-Other-Header": [ - "foo" - ], + "Content-Type": "text/html", + "Cookie": "c1=v1, c2=v2", + "Some-Other-Header": "foo", "User-Agent": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla Chrome Edge" @@ -672,9 +666,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ diff --git a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json index e37a5126355..8507cd1c33f 100644 --- a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json +++ b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json @@ -124,15 +124,9 @@ "bar", "baz" ], - "Content-Type": [ - "text/html" - ], - "Cookie": [ - "c1=v1, c2=v2" - ], - "Some-Other-Header": [ - "foo" - ], + "Content-Type": "text/html", + "Cookie": "c1=v1, c2=v2", + "Some-Other-Header": "foo", "User-Agent": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla Chrome Edge" @@ -156,9 +150,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ diff --git a/systemtest/approvals/TestIntake/UnknownSpanType.approved.json b/systemtest/approvals/TestIntake/UnknownSpanType.approved.json index 78b5df3636e..5f32c256529 100644 --- a/systemtest/approvals/TestIntake/UnknownSpanType.approved.json +++ b/systemtest/approvals/TestIntake/UnknownSpanType.approved.json @@ -253,18 +253,10 @@ "bar", "baz" ], - "Content-Type": [ - "text/html" - ], - "Cookie": [ - "c1=v1,c2=v2" - ], - "Some-Other-Header": [ - "foo" - ], - "User-Agent": [ - "Mozilla Chrome Edge" - ] + "Content-Type": "text/html", + "Cookie": "c1=v1,c2=v2", + "Some-Other-Header": "foo", + "User-Agent": "Mozilla Chrome Edge" } ], "http.request.method": [ @@ -278,9 +270,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.headers_sent": [ diff --git a/systemtest/approvals/TestRUMRoutingIntegration.approved.json b/systemtest/approvals/TestRUMRoutingIntegration.approved.json index 4f3e5090cb7..f65562fbe8a 100644 --- a/systemtest/approvals/TestRUMRoutingIntegration.approved.json +++ b/systemtest/approvals/TestRUMRoutingIntegration.approved.json @@ -1416,9 +1416,7 @@ ], "http.request.headers": [ { - "Accept": [ - "application/json" - ] + "Accept": "application/json" } ], "http.request.method": [ @@ -1435,9 +1433,7 @@ ], "http.response.headers": [ { - "Content-Type": [ - "application/json" - ] + "Content-Type": "application/json" } ], "http.response.status_code": [ diff --git a/systemtest/benchtest/profiles.go b/systemtest/benchtest/profiles.go index 9e2ee89b43a..0e0dcf515c8 100644 --- a/systemtest/benchtest/profiles.go +++ b/systemtest/benchtest/profiles.go @@ -18,9 +18,11 @@ package benchtest import ( + "compress/gzip" "context" "fmt" "io" + "math/rand/v2" "net/http" "os" "strconv" @@ -88,12 +90,30 @@ func (p *profiles) recordCPU() error { if benchConfig.CPUProfile == "" { return nil } - duration := 2 * benchConfig.Benchtime - profile, err := fetchProfile("/debug/pprof/profile", duration) - if err != nil { - return fmt.Errorf("failed to fetch CPU profile: %w", err) + // Limit profiling time to random 5% of overall time. + // This should not seriously affect the profile quality, + // since we merge the final profile form multiple sources, + // but prevent profile size from swelling. + var done bool + const tickets = 20 + duration := benchConfig.Benchtime / tickets + for i := range tickets { + if done || (rand.N(tickets-i)+i+1) < tickets { + time.Sleep(duration) + continue + } + profile, err := fetchProfile("/debug/pprof/profile", duration) + if err != nil { + return fmt.Errorf("failed to fetch CPU profile: %w", err) + } + // We don't need the address in the profile, so discard it to reduce the size. + if err := profile.Aggregate(true, true, true, true, false); err != nil { + return fmt.Errorf("failed to fetch CPU profile: %w", err) + } + profile = profile.Compact() + p.cpu = append(p.cpu, profile) + done = true } - p.cpu = append(p.cpu, profile) return nil } @@ -168,14 +188,15 @@ func (p *profiles) writeDeltas(filename string, deltas []*profile.Profile) error return err } defer f.Close() - return merged.Write(f) + w, err := gzip.NewWriterLevel(f, gzip.BestCompression) + if err != nil { + return err + } + defer w.Close() + return merged.WriteUncompressed(w) } func (p *profiles) mergeBenchmarkProfiles(profiles []*profile.Profile) (*profile.Profile, error) { - for i, profile := range profiles { - benchmarkName := p.benchmarkNames[i] - profile.SetLabel("benchmark", []string{benchmarkName}) - } merged, err := profile.Merge(profiles) if err != nil { return nil, fmt.Errorf("error merging profiles: %w", err) diff --git a/systemtest/cmd/apmbench/main.go b/systemtest/cmd/apmbench/main.go index 26a8df18e6c..075cc1b4849 100644 --- a/systemtest/cmd/apmbench/main.go +++ b/systemtest/cmd/apmbench/main.go @@ -22,6 +22,7 @@ import ( "flag" "fmt" "log" + "runtime/debug" "testing" "time" @@ -30,6 +31,7 @@ import ( "golang.org/x/time/rate" "github.com/elastic/apm-server/systemtest/benchtest" + "github.com/elastic/go-sysinfo" ) func Benchmark1000Transactions(b *testing.B, l *rate.Limiter) { @@ -138,6 +140,11 @@ func Benchmark10000AggregationGroups(b *testing.B, l *rate.Limiter) { func main() { flag.Parse() + bytes, err := sysMemory() + if err != nil { + log.Fatal(err) + } + debug.SetMemoryLimit(int64(float64(bytes) * 0.9)) if err := benchtest.Run( Benchmark1000Transactions, BenchmarkOTLPTraces, @@ -151,3 +158,15 @@ func main() { log.Fatal(err) } } + +func sysMemory() (uint64, error) { + host, err := sysinfo.Host() + if err != nil { + return 0, err + } + mem, err := host.Memory() + if err != nil { + return 0, err + } + return mem.Total, nil +} diff --git a/systemtest/go.mod b/systemtest/go.mod index 0f7fa76e55e..b2307b7a631 100644 --- a/systemtest/go.mod +++ b/systemtest/go.mod @@ -9,16 +9,17 @@ require ( github.com/elastic/apm-tools v0.0.0-20240607105915-a4f490dc6959 github.com/elastic/go-elasticsearch/v8 v8.15.0 github.com/elastic/go-lumber v0.1.1 + github.com/elastic/go-sysinfo v1.11.1 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 github.com/hashicorp/go-multierror v1.1.1 - github.com/jaegertracing/jaeger v1.61.0 + github.com/jaegertracing/jaeger v1.62.0 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.33.0 - github.com/tidwall/gjson v1.17.3 + github.com/tidwall/gjson v1.18.0 go.elastic.co/apm/v2 v2.6.2 go.elastic.co/fastjson v1.4.0 - go.opentelemetry.io/collector/pdata v1.16.0 - go.opentelemetry.io/collector/semconv v0.110.0 + go.opentelemetry.io/collector/pdata v1.17.0 + go.opentelemetry.io/collector/semconv v0.111.0 go.opentelemetry.io/otel v1.30.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 @@ -31,9 +32,9 @@ require ( go.opentelemetry.io/otel/trace v1.30.0 go.uber.org/zap v1.27.0 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.25.0 - golang.org/x/time v0.6.0 - google.golang.org/grpc v1.67.0 + golang.org/x/sys v0.26.0 + golang.org/x/time v0.7.0 + google.golang.org/grpc v1.67.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -51,7 +52,6 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect - github.com/elastic/go-sysinfo v1.11.1 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -65,7 +65,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.10 // indirect github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -95,9 +95,9 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/text v0.19.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/systemtest/go.sum b/systemtest/go.sum index 279ef79091d..f0817660e3d 100644 --- a/systemtest/go.sum +++ b/systemtest/go.sum @@ -77,8 +77,8 @@ github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jaegertracing/jaeger v1.61.0 h1:9PCP5vkilmoxmSHa9jFvtULoduFJqJ0/bHbRxUMPWTc= -github.com/jaegertracing/jaeger v1.61.0/go.mod h1:DMy9PNQ7tOgo811jOv7UAQaM0NeSJ95lh6SW3O1s1Xk= +github.com/jaegertracing/jaeger v1.62.0 h1:YoaJ2e8oVz5sqGGlVAKSUCED8DzJ1q7PojBmZFNKoJA= +github.com/jaegertracing/jaeger v1.62.0/go.mod h1:jhEIHazwyb+a6xlRBi+p96BAvTYTSmGkghcwdQfV7FM= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= @@ -87,8 +87,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= +github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -156,8 +156,8 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= -github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -177,10 +177,10 @@ go.elastic.co/apm/v2 v2.6.2 h1:VBplAxgbOgTv+Giw/FS91xJpHYw/q8fz/XKPvqC+7/o= go.elastic.co/apm/v2 v2.6.2/go.mod h1:33rOXgtHwbgZcDgi6I/GtCSMZQqgxkHC0IQT3gudKvo= go.elastic.co/fastjson v1.4.0 h1:a4BXUKXZHAzjVOPrqtEx2FDsIRBCMek01vCnrtyutWs= go.elastic.co/fastjson v1.4.0/go.mod h1:ZD5um63l0/8TIdddZbL2znD83FAr2IckYa3KR7VcdNA= -go.opentelemetry.io/collector/pdata v1.16.0 h1:g02K8jlRnmQ7TQDuXpdgVL6vIxIVqr5Gbb1qIR27rto= -go.opentelemetry.io/collector/pdata v1.16.0/go.mod h1:YZZJIt2ehxosYf/Y1pbvexjNWsIGNNrzzlCTO9jC1F4= -go.opentelemetry.io/collector/semconv v0.110.0 h1:KHQnOHe3gUz0zsxe8ph9kN5OTypCFD4V+06AiBTfeNk= -go.opentelemetry.io/collector/semconv v0.110.0/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A= +go.opentelemetry.io/collector/pdata v1.17.0 h1:z8cjjT2FThAehWu5fbF48OnZyK5q8xd1UhC4XszDo0w= +go.opentelemetry.io/collector/pdata v1.17.0/go.mod h1:yZaQ9KZAm/qie96LTygRKxOXMq0/54h8OW7330ycuvQ= +go.opentelemetry.io/collector/semconv v0.111.0 h1:ELleMtLBzeZ3xhfhYPmFcLc0hJMqRxhOB0eY60WLivw= +go.opentelemetry.io/collector/semconv v0.111.0/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= @@ -214,16 +214,16 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -241,16 +241,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -263,8 +263,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/systemtest/monitoring_test.go b/systemtest/monitoring_test.go index 354ffcc6ffb..eae6541afd5 100644 --- a/systemtest/monitoring_test.go +++ b/systemtest/monitoring_test.go @@ -141,7 +141,7 @@ func getBeatsMonitoring(t testing.TB, srv *apmservertest.Server, type_ string, o espoll.TermQuery{Field: type_ + ".beat.uuid", Value: srv.BeatUUID}, ).WithSort("timestamp:desc") if _, err := req.Do(context.Background(), &result, espoll.WithCondition(result.Hits.MinHitsCondition(1))); err != nil { - t.Fatal(err) + t.Error(err) } var doc beatsMonitoringDoc diff --git a/testing/benchmark/Makefile b/testing/benchmark/Makefile index cf894325b58..9baf132edaf 100644 --- a/testing/benchmark/Makefile +++ b/testing/benchmark/Makefile @@ -2,6 +2,12 @@ APMBENCH_PATH ?= ../../systemtest/cmd/apmbench APMBENCH_GOOS ?= linux APMBENCH_GOARCH ?= amd64 +MOXY_GOOS ?= linux +MOXY_GOARCH ?= amd64 + +APM_SERVER_GOOS ?= linux +APM_SERVER_GOARCH ?= amd64 + TFVARS_SOURCE ?= terraform.tfvars.example BENCHMARK_WARMUP_TIME ?= 5m @@ -23,6 +29,7 @@ SSH_USER ?= ec2-user SSH_OPTS ?= -o LogLevel=ERROR -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10 SSH_KEY ?= ~/.ssh/id_rsa_terraform WORKER_IP = $(shell terraform output -raw public_ip) +APM_SERVER_IP = $(shell terraform output -raw apm_server_ip) SHELL = /bin/bash .SHELLFLAGS = -o pipefail -c @@ -67,6 +74,15 @@ apmbench: @echo "-> Building apmbench..." @cd $(APMBENCH_PATH) && CGO_ENABLED=0 GOOS=$(APMBENCH_GOOS) GOARCH=$(APMBENCH_GOARCH) go build . +.PHONY: moxy +moxy: + @echo "-> Building moxy..." + @cd ../../tools && CGO_ENABLED=0 GOOS=$(MOXY_GOOS) GOARCH=$(MOXY_GOARCH) go build -o "../build" github.com/elastic/apm-perf/cmd/moxy + +.PHONY: apm-server +apm-server: + @cd ../.. && make build/apm-server-$(APM_SERVER_GOOS)-$(APM_SERVER_GOARCH) && mv build/apm-server-$(APM_SERVER_GOOS)-$(APM_SERVER_GOARCH) build/apm-server + .PHONY: init init: @terraform init @@ -110,6 +126,12 @@ index-benchmark-results: _default-gobench-vars .PHONY: _default-gobench-vars _default-gobench-vars: +ifeq ($(RUN_STANDALONE),true) + $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),apm_server_size=$(shell echo var.standalone_apm_server_instance_size | terraform console | tr -d '"')) + $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),moxy_size=$(shell echo var.standalone_moxy_instance_size | terraform console | tr -d '"')) + $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),build_sha=$(shell git rev-parse HEAD)) + $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),bench_mode=standalone) +else # TODO(marclop) Update code below to use a foor loop, rather than copying the lines. $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),apm_server_size=$(shell echo var.apm_server_size | terraform console | tr -d '"')) $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),elasticsearch_size=$(shell echo var.elasticsearch_size | terraform console | tr -d '"')) @@ -117,6 +139,12 @@ _default-gobench-vars: $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),apm_server_zone_count=$(shell echo var.apm_server_zone_count | terraform console | tr -d '"')) $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),elasticsearch_zone_count=$(shell echo var.elasticsearch_zone_count | terraform console | tr -d '"')) $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),build_sha=$(shell curl -sL -H "Authorization: Bearer $(shell terraform output -raw apm_secret_token )" $(shell terraform output -raw apm_server_url ) | jq -r '.build_sha')) + $(eval GOBENCH_DEFAULT_TAGS = $(GOBENCH_DEFAULT_TAGS),bench_mode=cloud) +endif + +.PHONY: cat-apm-server-logs +cat-apm-server-logs: + @ssh $(SSH_OPTS) -i $(SSH_KEY) $(SSH_USER)@$(APM_SERVER_IP) "cat /var/log/apm-server/*" $(SSH_KEY): @ssh-keygen -t rsa -b 4096 -C "$(USER)@elastic.co" -N "" -f $(SSH_KEY) @@ -172,4 +200,4 @@ elastic_agent_docker_image: build_elastic_agent_docker_image build_elastic_agent_docker_image: @env BASE_IMAGE=${ELASTIC_AGENT_DOCKER_IMAGE}:${ELASTIC_AGENT_IMAGE_TAG} GOARCH=amd64 \ bash ${REPO_ROOT}/testing/docker/elastic-agent/build.sh \ - -t ${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG} + -t ${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG} \ No newline at end of file diff --git a/testing/benchmark/README.md b/testing/benchmark/README.md index ab2f89c6aa1..c3cf50297e6 100644 --- a/testing/benchmark/README.md +++ b/testing/benchmark/README.md @@ -89,7 +89,7 @@ overridden automatically, you need to remove it manually if present. #### Override docker image tag It is possible to override the tag of the docker image that is run in the remote ESS deployment. You can -specify any of the avilable tags (such as `8.3.0-SNAPSHOT` or a more specific tag `8.3.0-c655cda8-SNAPSHOT`). +specify any of the available tags (such as `8.3.0-SNAPSHOT` or a more specific tag `8.3.0-c655cda8-SNAPSHOT`). Alternatively, you can run `make docker-override-committed-version` in your shell, to have use the committed tags in the `docker-compose.yml` file in the repository root. diff --git a/testing/benchmark/main.tf b/testing/benchmark/main.tf index e699f56485e..4b888e7a78a 100644 --- a/testing/benchmark/main.tf +++ b/testing/benchmark/main.tf @@ -45,7 +45,46 @@ locals { name_prefix = "${coalesce(var.user_name, "unknown-user")}-bench" } +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.14.0" + + name = "${var.user_name}-worker" + cidr = var.vpc_cidr + + azs = ["${var.worker_region}a"] + public_subnets = var.public_cidr + enable_ipv6 = false + enable_nat_gateway = false + single_nat_gateway = false + + manage_default_security_group = true + default_security_group_ingress = [ + { + "from_port" : 0, + "to_port" : 0, + "protocol" : -1, + "self" : true, + "cidr_blocks" : "0.0.0.0/0", + } + ] + default_security_group_egress = [ + { + "from_port" : 0, + "to_port" : 0, + "protocol" : -1, + "cidr_blocks" : "0.0.0.0/0", + } + ] + + tags = merge(local.ci_tags, module.tags.tags) + vpc_tags = { + Name = "vpc-${var.user_name}-worker" + } +} + module "ec_deployment" { + count = var.run_standalone ? 0 : 1 source = "../infra/terraform/modules/ec_deployment" region = var.ess_region @@ -73,12 +112,13 @@ module "ec_deployment" { module "benchmark_worker" { source = "../infra/terraform/modules/benchmark_executor" - region = var.worker_region + vpc_id = module.vpc.vpc_id + region = var.worker_region user_name = var.user_name - apm_server_url = module.ec_deployment.apm_url - apm_secret_token = module.ec_deployment.apm_secret_token + apm_server_url = var.run_standalone ? module.standalone_apm_server[0].apm_server_url : module.ec_deployment[0].apm_url + apm_secret_token = var.run_standalone ? module.standalone_apm_server[0].apm_secret_token : module.ec_deployment[0].apm_secret_token apmbench_bin_path = var.apmbench_bin_path instance_type = var.worker_instance_type @@ -86,5 +126,41 @@ module "benchmark_worker" { public_key = var.public_key private_key = var.private_key - tags = merge(local.ci_tags, module.tags.tags) + tags = merge(local.ci_tags, module.tags.tags) + depends_on = [module.standalone_apm_server, module.ec_deployment] +} + +module "moxy" { + count = var.run_standalone ? 1 : 0 + source = "../infra/terraform/modules/moxy" + + vpc_id = module.vpc.vpc_id + instance_type = var.standalone_moxy_instance_size + moxy_bin_path = var.moxy_bin_path + + aws_provisioner_key_name = var.private_key + + tags = merge(local.ci_tags, module.tags.tags) + depends_on = [module.vpc] +} + + +module "standalone_apm_server" { + count = var.run_standalone ? 1 : 0 + source = "../infra/terraform/modules/standalone_apm_server" + + vpc_id = module.vpc.vpc_id + aws_os = "amzn2-ami-hvm-*-x86_64-ebs" + apm_instance_type = var.standalone_apm_server_instance_size + apm_server_bin_path = var.apm_server_bin_path + ea_managed = false + + aws_provisioner_key_name = var.private_key + + elasticsearch_url = module.moxy[0].moxy_url + elasticsearch_username = "elastic" + elasticsearch_password = module.moxy[0].moxy_password + + tags = merge(local.ci_tags, module.tags.tags) + depends_on = [module.moxy] } diff --git a/testing/benchmark/outputs.tf b/testing/benchmark/outputs.tf index 371d9c43760..2cad7994126 100644 --- a/testing/benchmark/outputs.tf +++ b/testing/benchmark/outputs.tf @@ -4,38 +4,45 @@ output "public_ip" { } output "elasticsearch_url" { - value = module.ec_deployment.elasticsearch_url + value = var.run_standalone ? module.moxy[0].moxy_url : module.ec_deployment[0].elasticsearch_url description = "The secure Elasticsearch URL" } output "elasticsearch_username" { - value = module.ec_deployment.elasticsearch_username + value = var.run_standalone ? "elastic" : module.ec_deployment[0].elasticsearch_username description = "The Elasticsearch username" sensitive = true } output "elasticsearch_password" { - value = module.ec_deployment.elasticsearch_password + value = var.run_standalone ? module.moxy[0].moxy_password : module.ec_deployment[0].elasticsearch_password description = "The Elasticsearch password" sensitive = true } output "kibana_url" { - value = module.ec_deployment.kibana_url + value = var.run_standalone ? "" : module.ec_deployment[0].kibana_url description = "The secure Kibana URL" } + output "apm_secret_token" { - value = module.ec_deployment.apm_secret_token + value = var.run_standalone ? module.standalone_apm_server[0].apm_secret_token : module.ec_deployment[0].apm_secret_token description = "The APM Server secret token" sensitive = true } output "apm_server_url" { - value = module.ec_deployment.apm_url + value = var.run_standalone ? module.standalone_apm_server[0].apm_server_url : module.ec_deployment[0].apm_url description = "The APM Server URL" + sensitive = true +} + +output "apm_server_ip" { + value = var.run_standalone ? module.standalone_apm_server[0].apm_server_ip : "" + description = "The APM Server EC2 IP address" } output "admin_console_url" { - value = module.ec_deployment.admin_console_url + value = var.run_standalone ? "" : module.ec_deployment[0].admin_console_url description = "The admin console URL" } diff --git a/testing/benchmark/system-profiles/16GBx2zone.tfvars b/testing/benchmark/system-profiles/16GBx2zone.tfvars index d081604f332..84da1c4878b 100644 --- a/testing/benchmark/system-profiles/16GBx2zone.tfvars +++ b/testing/benchmark/system-profiles/16GBx2zone.tfvars @@ -1,5 +1,11 @@ user_name = "USER" +# APM bench + +worker_instance_type = "c6i.2xlarge" + +# Elastic Cloud + # The number of AZs the APM Server should span. apm_server_zone_count = 1 # The Elasticsearch cluster node size. @@ -10,5 +16,8 @@ elasticsearch_zone_count = 2 apm_server_size = "16g" # Number of shards for the ES indices apm_shards = 4 -# Benchmarks executor size executor -worker_instance_type = "c6i.2xlarge" + +# Standalone + +standalone_apm_server_instance_size = "c6i.2xlarge" +standalone_moxy_instance_size = "c6i.4xlarge" diff --git a/testing/benchmark/system-profiles/1GBx1zone.tfvars b/testing/benchmark/system-profiles/1GBx1zone.tfvars index a2ca3dac002..8b1ff546e60 100644 --- a/testing/benchmark/system-profiles/1GBx1zone.tfvars +++ b/testing/benchmark/system-profiles/1GBx1zone.tfvars @@ -1,5 +1,11 @@ user_name = "USER" +# APM bench + +worker_instance_type = "c6i.large" + +# Elastic Cloud + # The number of AZs the APM Server should span. apm_server_zone_count = 1 # The Elasticsearch cluster node size. @@ -8,3 +14,8 @@ elasticsearch_size = "16g" elasticsearch_zone_count = 2 # APM server instance size apm_server_size = "1g" + +# Standalone + +standalone_apm_server_instance_size = "c6i.large" +standalone_moxy_instance_size = "c6i.xlarge" diff --git a/testing/benchmark/system-profiles/2GBx1zone.tfvars b/testing/benchmark/system-profiles/2GBx1zone.tfvars index 668f12f9edf..a3114b4b989 100644 --- a/testing/benchmark/system-profiles/2GBx1zone.tfvars +++ b/testing/benchmark/system-profiles/2GBx1zone.tfvars @@ -1,5 +1,11 @@ user_name = "USER" +# APM bench + +worker_instance_type = "c6i.large" + +# Elastic Cloud + # The number of AZs the APM Server should span. apm_server_zone_count = 1 # The Elasticsearch cluster node size. @@ -7,4 +13,9 @@ elasticsearch_size = "16g" # The number of AZs the Elasticsearch cluster should have. elasticsearch_zone_count = 2 # APM server instance size -apm_server_size = "2g" \ No newline at end of file +apm_server_size = "2g" + +# Standalone + +standalone_apm_server_instance_size = "c6i.large" +standalone_moxy_instance_size = "c6i.xlarge" diff --git a/testing/benchmark/system-profiles/32GBx2zone.tfvars b/testing/benchmark/system-profiles/32GBx2zone.tfvars index 10a9180257b..67cc51afc2b 100644 --- a/testing/benchmark/system-profiles/32GBx2zone.tfvars +++ b/testing/benchmark/system-profiles/32GBx2zone.tfvars @@ -1,5 +1,11 @@ user_name = "USER" +# APM bench + +worker_instance_type = "c6i.2xlarge" + +# Elastic Cloud + # The number of AZs the APM Server should span. apm_server_zone_count = 1 # The Elasticsearch cluster node size. @@ -12,5 +18,8 @@ elasticsearch_dedicated_masters = true apm_server_size = "32g" # Number of shards for the ES indices apm_shards = 4 -# Benchmarks executor size executor -worker_instance_type = "c6i.2xlarge" + +# Standalone + +standalone_apm_server_instance_size = "c6i.4xlarge" +standalone_moxy_instance_size = "c6i.8xlarge" diff --git a/testing/benchmark/system-profiles/4GBx1zone.tfvars b/testing/benchmark/system-profiles/4GBx1zone.tfvars index f55f9099444..3ffda6cdc31 100644 --- a/testing/benchmark/system-profiles/4GBx1zone.tfvars +++ b/testing/benchmark/system-profiles/4GBx1zone.tfvars @@ -1,5 +1,11 @@ user_name = "USER" +# APM bench + +worker_instance_type = "c6i.large" + +# Elastic Cloud + # The number of AZs the APM Server should span. apm_server_zone_count = 1 # The Elasticsearch cluster node size. @@ -7,4 +13,9 @@ elasticsearch_size = "32g" # The number of AZs the Elasticsearch cluster should have. elasticsearch_zone_count = 2 # APM server instance size -apm_server_size = "4g" \ No newline at end of file +apm_server_size = "4g" + +# Standalone + +standalone_apm_server_instance_size = "c6i.large" +standalone_moxy_instance_size = "c6i.xlarge" diff --git a/testing/benchmark/system-profiles/8GBx1zone.tfvars b/testing/benchmark/system-profiles/8GBx1zone.tfvars index 62719a89b15..b3b42e74409 100644 --- a/testing/benchmark/system-profiles/8GBx1zone.tfvars +++ b/testing/benchmark/system-profiles/8GBx1zone.tfvars @@ -1,5 +1,11 @@ user_name = "USER" +# APM bench + +worker_instance_type = "c6i.xlarge" + +# Elastic Cloud + # The number of AZs the APM Server should span. apm_server_zone_count = 1 # The Elasticsearch cluster node size. @@ -8,5 +14,8 @@ elasticsearch_size = "64g" elasticsearch_zone_count = 2 # APM server instance size apm_server_size = "8g" -# Benchmarks executor size executor -worker_instance_type = "c6i.2xlarge" + +# Standalone + +standalone_apm_server_instance_size = "c6i.xlarge" +standalone_moxy_instance_size = "c6i.2xlarge" diff --git a/testing/benchmark/terraform.tfvars.example b/testing/benchmark/terraform.tfvars.example index d58973b61b1..2b44a828d6c 100644 --- a/testing/benchmark/terraform.tfvars.example +++ b/testing/benchmark/terraform.tfvars.example @@ -67,3 +67,9 @@ user_name = "USER" # Override the default shard settings for APM indices. Defaults to 0, which doesn't # change the default shard settings. # apm_shards = 12 + +# Override the default APM Server VM size in standalone bench mode. +# standalone_apm_server_instance_size = "c6i.2xlarge" + +# Override the default Moxy VM size in standalone bench mode. +# standalone_moxy_instance_size = "c6i.4xlarge" \ No newline at end of file diff --git a/testing/benchmark/variables.tf b/testing/benchmark/variables.tf index deb2d05d6b0..b06a4d7a6b2 100644 --- a/testing/benchmark/variables.tf +++ b/testing/benchmark/variables.tf @@ -5,6 +5,12 @@ variable "user_name" { type = string } +variable "run_standalone" { + default = false + description = "If set run benchmarks against standalone APM Server connected to moxy" + type = bool +} + ## Deployment configuration variable "ess_region" { @@ -86,7 +92,49 @@ variable "drop_pipeline" { type = bool } -## Worker configuraiton +# Standalone + +variable "apm_server_bin_path" { + default = "../../build" + type = string + description = "Optional path to APM Server binary" +} + +variable "moxy_bin_path" { + default = "../../build" + type = string + description = "Optional path to moxy binary" +} + +variable "standalone_apm_server_instance_size" { + default = "c6i.2xlarge" + type = string + description = "Optional instance type to use for APM Server VM" +} + +variable "standalone_moxy_instance_size" { + default = "c6i.4xlarge" + type = string + description = "Optional instance type to use for moxy VM" +} + +## VPC Network settings + +variable "vpc_cidr" { + default = "192.168.44.0/24" + type = string +} + +variable "public_cidr" { + default = [ + "192.168.44.0/26", + "192.168.44.64/26", + "192.168.44.128/26", + ] + type = list(string) +} + +## Worker configuration variable "worker_region" { default = "us-west-2" diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index b4dd24101d1..f1150ddbb51 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.16.0-2dd49c28-SNAPSHOT + version: 8.16.0-644441e6-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index 0a85e504744..32b09650344 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.16.0-2dd49c28-SNAPSHOT + version: 8.16.0-644441e6-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index b9b86591b68..f2b0defab02 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.16.0-2dd49c28-SNAPSHOT + version: 8.16.0-644441e6-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch diff --git a/testing/infra/terraform/modules/benchmark_executor/README.md b/testing/infra/terraform/modules/benchmark_executor/README.md index 315861b6416..6eca6317a74 100644 --- a/testing/infra/terraform/modules/benchmark_executor/README.md +++ b/testing/infra/terraform/modules/benchmark_executor/README.md @@ -17,7 +17,6 @@ so that `apmbench` can be run against a configured APM Server. | Name | Source | Version | |------|--------|---------| | [ec2\_instance](#module\_ec2\_instance) | terraform-aws-modules/ec2-instance/aws | 3.5.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | 3.14.0 | ## Resources @@ -28,6 +27,8 @@ so that `apmbench` can be run against a configured APM Server. | [null_resource.apmbench](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.envrc](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [aws_ami.worker_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_security_group.security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/security_group) | data source | +| [aws_subnets.public_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -38,12 +39,11 @@ so that `apmbench` can be run against a configured APM Server. | [apmbench\_bin\_path](#input\_apmbench\_bin\_path) | Optionally upload the apmbench binary from the specified path to the worker machine | `string` | `""` | no | | [instance\_type](#input\_instance\_type) | Optional instance type to use for the worker VM | `string` | `"c6i.large"` | no | | [private\_key](#input\_private\_key) | n/a | `string` | `"~/.ssh/id_rsa_terraform"` | no | -| [public\_cidr](#input\_public\_cidr) | n/a | `list(string)` |
[
"192.168.44.0/26",
"192.168.44.64/26",
"192.168.44.128/26"
]
| no | | [public\_key](#input\_public\_key) | n/a | `string` | `"~/.ssh/id_rsa_terraform.pub"` | no | | [region](#input\_region) | n/a | `string` | `"us-west2"` | no | | [tags](#input\_tags) | Optional set of tags to use for all resources | `map(string)` | `{}` | no | | [user\_name](#input\_user\_name) | Required username to use for resource name prefixes | `string` | n/a | yes | -| [vpc\_cidr](#input\_vpc\_cidr) | n/a | `string` | `"192.168.44.0/24"` | no | +| [vpc\_id](#input\_vpc\_id) | VPC ID to provision the EC2 instance | `string` | n/a | yes | ## Outputs diff --git a/testing/infra/terraform/modules/benchmark_executor/instance.tf b/testing/infra/terraform/modules/benchmark_executor/instance.tf index 18d55462c66..7b41f643ebf 100644 --- a/testing/infra/terraform/modules/benchmark_executor/instance.tf +++ b/testing/infra/terraform/modules/benchmark_executor/instance.tf @@ -6,50 +6,6 @@ locals { } } -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "3.14.0" - - name = "${var.user_name}-worker" - cidr = var.vpc_cidr - - azs = [for letter in ["a", "b", "c"] : "${var.region}${letter}"] - public_subnets = var.public_cidr - enable_ipv6 = false - enable_nat_gateway = false - single_nat_gateway = false - - manage_default_security_group = true - default_security_group_ingress = [ - { - "from_port" : 0, - "to_port" : 0, - "protocol" : -1, - "self" : true, - "cidr_blocks" : "0.0.0.0/0", - } - ] - default_security_group_egress = [ - { - "from_port" : 0, - "to_port" : 0, - "protocol" : -1, - "cidr_blocks" : "0.0.0.0/0", - } - ] - - tags = merge(var.tags, local.ec2_tags) - vpc_tags = { - Name = "vpc-${var.user_name}-worker" - } -} - -resource "aws_key_pair" "worker" { - key_name = "${var.user_name}_worker_key" - public_key = file(var.public_key) - tags = merge(var.tags, local.ec2_tags) -} - data "aws_ami" "worker_ami" { owners = ["amazon"] most_recent = true @@ -60,6 +16,18 @@ data "aws_ami" "worker_ami" { } } +data "aws_subnets" "public_subnets" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +data "aws_security_group" "security_group" { + vpc_id = var.vpc_id + name = "default" +} + module "ec2_instance" { source = "terraform-aws-modules/ec2-instance/aws" @@ -68,9 +36,15 @@ module "ec2_instance" { ami = data.aws_ami.worker_ami.id instance_type = var.instance_type monitoring = false - vpc_security_group_ids = [module.vpc.default_security_group_id] - subnet_id = module.vpc.public_subnets[0] + vpc_security_group_ids = [data.aws_security_group.security_group.id] + subnet_id = data.aws_subnets.public_subnets.ids[0] associate_public_ip_address = true key_name = aws_key_pair.worker.id tags = merge(var.tags, local.ec2_tags) } + +resource "aws_key_pair" "worker" { + key_name = "${var.user_name}_worker_key" + public_key = file(var.public_key) + tags = merge(var.tags, local.ec2_tags) +} diff --git a/testing/infra/terraform/modules/benchmark_executor/variables.tf b/testing/infra/terraform/modules/benchmark_executor/variables.tf index 5edc17f5bae..79e21837e5e 100644 --- a/testing/infra/terraform/modules/benchmark_executor/variables.tf +++ b/testing/infra/terraform/modules/benchmark_executor/variables.tf @@ -18,6 +18,11 @@ variable "instance_type" { description = "Optional instance type to use for the worker VM" } +variable "vpc_id" { + description = "VPC ID to provision the EC2 instance" + type = string +} + variable "apm_secret_token" { default = "" type = string @@ -39,22 +44,6 @@ variable "tags" { description = "Optional set of tags to use for all resources" } -## VPC Network settings - -variable "vpc_cidr" { - default = "192.168.44.0/24" - type = string -} - -variable "public_cidr" { - default = [ - "192.168.44.0/26", - "192.168.44.64/26", - "192.168.44.128/26", - ] - type = list(string) -} - variable "region" { default = "us-west2" type = string diff --git a/testing/infra/terraform/modules/moxy/README.md b/testing/infra/terraform/modules/moxy/README.md new file mode 100644 index 00000000000..68935f1f6b8 --- /dev/null +++ b/testing/infra/terraform/modules/moxy/README.md @@ -0,0 +1,40 @@ + +# Moxy module + +This module can be used to create a stub ElasticSearch deployment (Moxy). + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [random](#provider\_random) | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_instance.moxy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance) | resource | +| [aws_key_pair.provisioner_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | +| [aws_security_group.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [random_password.moxy_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [aws_ami.worker_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_subnets.public_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_provisioner\_key\_name](#input\_aws\_provisioner\_key\_name) | ssh key name to create the aws key pair and remote provision the EC2 instance | `string` | n/a | yes | +| [instance\_type](#input\_instance\_type) | Moxy instance type | `string` | n/a | yes | +| [moxy\_bin\_path](#input\_moxy\_bin\_path) | Moxy path to binary to copy to the worker machine | `string` | n/a | yes | +| [tags](#input\_tags) | Optional set of tags to use for all resources | `map(string)` | `{}` | no | +| [vpc\_id](#input\_vpc\_id) | VPC ID to provision the EC2 instance | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [moxy\_password](#output\_moxy\_password) | Moxy server password | +| [moxy\_url](#output\_moxy\_url) | Moxy server URL | + \ No newline at end of file diff --git a/testing/infra/terraform/modules/moxy/header.md b/testing/infra/terraform/modules/moxy/header.md new file mode 100644 index 00000000000..519b5d9b58b --- /dev/null +++ b/testing/infra/terraform/modules/moxy/header.md @@ -0,0 +1,3 @@ +# Moxy module + +This module can be used to create a stub ElasticSearch deployment (Moxy). diff --git a/testing/infra/terraform/modules/moxy/main.tf b/testing/infra/terraform/modules/moxy/main.tf new file mode 100644 index 00000000000..245b69f0987 --- /dev/null +++ b/testing/infra/terraform/modules/moxy/main.tf @@ -0,0 +1,103 @@ +locals { + moxy_port = "9200" + bin_path = "/tmp/moxy" +} + +data "aws_ami" "worker_ami" { + owners = ["amazon"] + most_recent = true + + filter { + name = "name" + values = ["amzn2-ami-hvm-*-x86_64-ebs"] + } +} + +data "aws_subnets" "public_subnets" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +resource "aws_security_group" "main" { + vpc_id = var.vpc_id + egress = [ + { + cidr_blocks = ["0.0.0.0/0"] + description = "" + from_port = 0 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "-1" + security_groups = [] + self = false + to_port = 0 + } + ] + ingress = [ + { + cidr_blocks = ["0.0.0.0/0"] + description = "" + from_port = 22 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "tcp" + security_groups = [] + self = false + to_port = 22 + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "" + from_port = local.moxy_port + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "tcp" + security_groups = [] + self = false + to_port = local.moxy_port + } + ] +} + +resource "aws_instance" "moxy" { + ami = data.aws_ami.worker_ami.id + instance_type = var.instance_type + subnet_id = data.aws_subnets.public_subnets.ids[0] + vpc_security_group_ids = [aws_security_group.main.id] + key_name = aws_key_pair.provisioner_key.key_name + monitoring = false + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_ip + private_key = file("${var.aws_provisioner_key_name}") + } + + provisioner "file" { + source = "${var.moxy_bin_path}/moxy" + destination = local.bin_path + } + provisioner "remote-exec" { + inline = [ + "sudo cp ${local.bin_path} moxy", + "sudo chmod +x moxy", + "screen -d -m ./moxy -port=${local.moxy_port} -password=${random_password.moxy_password.result}", + "sleep 1" + ] + } + + tags = var.tags +} + +resource "aws_key_pair" "provisioner_key" { + public_key = file("${var.aws_provisioner_key_name}.pub") + tags = var.tags +} + +resource "random_password" "moxy_password" { + length = 16 + special = false +} diff --git a/testing/infra/terraform/modules/moxy/outputs.tf b/testing/infra/terraform/modules/moxy/outputs.tf new file mode 100644 index 00000000000..ad4cb489bec --- /dev/null +++ b/testing/infra/terraform/modules/moxy/outputs.tf @@ -0,0 +1,10 @@ +output "moxy_url" { + value = "http://${aws_instance.moxy.public_ip}:${local.moxy_port}" + description = "Moxy server URL" +} + +output "moxy_password" { + value = random_password.moxy_password.result + description = "Moxy server password" + sensitive = true +} diff --git a/testing/infra/terraform/modules/moxy/variables.tf b/testing/infra/terraform/modules/moxy/variables.tf new file mode 100644 index 00000000000..4c8538b2661 --- /dev/null +++ b/testing/infra/terraform/modules/moxy/variables.tf @@ -0,0 +1,25 @@ +variable "instance_type" { + type = string + description = "Moxy instance type" +} + +variable "vpc_id" { + description = "VPC ID to provision the EC2 instance" + type = string +} + +variable "aws_provisioner_key_name" { + description = "ssh key name to create the aws key pair and remote provision the EC2 instance" + type = string +} + +variable "moxy_bin_path" { + type = string + description = "Moxy path to binary to copy to the worker machine" +} + +variable "tags" { + type = map(string) + default = {} + description = "Optional set of tags to use for all resources" +} diff --git a/testing/infra/terraform/modules/standalone_apm_server/README.md b/testing/infra/terraform/modules/standalone_apm_server/README.md new file mode 100644 index 00000000000..6b452b50cac --- /dev/null +++ b/testing/infra/terraform/modules/standalone_apm_server/README.md @@ -0,0 +1,53 @@ + +# Standalone APM Server module + +This module can be used to create a standalone APM Server deployment against a set of predefined architectures and instance types. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [external](#provider\_external) | n/a | +| [null](#provider\_null) | n/a | +| [random](#provider\_random) | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_instance.apm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance) | resource | +| [aws_key_pair.provisioner_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | +| [aws_security_group.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [null_resource.apm_server_log](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_password.apm_secret_token](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [aws_ami.os](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_subnets.public_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | +| [external_external.latest_apm_server](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | +| [external_external.latest_elastic_agent](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apm\_instance\_type](#input\_apm\_instance\_type) | Optional apm server instance type overide | `string` | `""` | no | +| [apm\_server\_bin\_path](#input\_apm\_server\_bin\_path) | Optionally use the apm-server binary from the specified path instead | `string` | `""` | no | +| [aws\_os](#input\_aws\_os) | Optional aws EC2 instance OS | `string` | `""` | no | +| [aws\_provisioner\_key\_name](#input\_aws\_provisioner\_key\_name) | ssh key name to create the aws key pair and remote provision the EC2 instance | `string` | n/a | yes | +| [ea\_managed](#input\_ea\_managed) | Whether or not install Elastic Agent managed APM Server | `bool` | `false` | no | +| [elasticsearch\_password](#input\_elasticsearch\_password) | The Elasticsearch password | `string` | n/a | yes | +| [elasticsearch\_url](#input\_elasticsearch\_url) | The secure Elasticsearch URL | `string` | n/a | yes | +| [elasticsearch\_username](#input\_elasticsearch\_username) | The Elasticsearch username | `string` | n/a | yes | +| [stack\_version](#input\_stack\_version) | Optional stack version | `string` | `"latest"` | no | +| [tags](#input\_tags) | Optional set of tags to use for all deployments | `map(string)` | `{}` | no | +| [vpc\_id](#input\_vpc\_id) | VPC ID to provision the EC2 instance | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [apm\_secret\_token](#output\_apm\_secret\_token) | The APM Server secret token | +| [apm\_server\_ip](#output\_apm\_server\_ip) | The APM Server EC2 IP address | +| [apm\_server\_url](#output\_apm\_server\_url) | The APM Server URL | + \ No newline at end of file diff --git a/testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl b/testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl index 5061811d904..e51007ccf5a 100644 --- a/testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl +++ b/testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl @@ -6,6 +6,11 @@ apm-server: secret_token: ${apm_secret_token} rum: enabled: true + expvar: + enabled: true + pprof: + enabled: true + output: elasticsearch: hosts: [ ${elasticsearch_url} ] diff --git a/testing/infra/terraform/modules/standalone_apm_server/header.md b/testing/infra/terraform/modules/standalone_apm_server/header.md new file mode 100644 index 00000000000..84009bb8dcc --- /dev/null +++ b/testing/infra/terraform/modules/standalone_apm_server/header.md @@ -0,0 +1,3 @@ +# Standalone APM Server module + +This module can be used to create a standalone APM Server deployment against a set of predefined architectures and instance types. diff --git a/testing/infra/terraform/modules/standalone_apm_server/main.tf b/testing/infra/terraform/modules/standalone_apm_server/main.tf index 251251181b0..e426fa0e5ce 100644 --- a/testing/infra/terraform/modules/standalone_apm_server/main.tf +++ b/testing/infra/terraform/modules/standalone_apm_server/main.tf @@ -6,6 +6,7 @@ locals { "debian-10-arm64" = "136693071363" # debian "debian-11-arm64" = "136693071363" # debian "amzn2-ami-kernel-5.10" = "137112412989" # amazon + "amzn2-ami-hvm-*-x86_64-ebs" = "137112412989" # amazon "al2023-ami-2023" = "137112412989" # amazon "RHEL-7" = "309956199498" # Red Hat "RHEL-8" = "309956199498" # Red Hat @@ -18,6 +19,7 @@ locals { "debian-10-arm64" = "t4g.nano" "debian-11-arm64" = "t4g.nano" "amzn2-ami-kernel-5.10" = "t4g.nano" + "amzn2-ami-hvm-*-x86_64-ebs" = "t4g.nano" "al2023-ami-2023" = "t4g.nano" "RHEL-7" = "t3a.micro" # RHEL-7 doesn't support arm "RHEL-8" = "t4g.micro" # RHEL doesn't support nano instances @@ -30,6 +32,7 @@ locals { "debian-10-arm64" = "arm64" "debian-11-arm64" = "arm64" "amzn2-ami-kernel-5.10" = "arm64" + "amzn2-ami-hvm-*-x86_64-ebs" = "x86_64" "al2023-ami-2023" = "arm64" "RHEL-7" = "x86_64" # RHEL-7 doesn't support arm "RHEL-8" = "arm64" @@ -66,13 +69,16 @@ locals { "debian-10-arm64" = "admin" "debian-11-arm64" = "admin" "amzn2-ami-kernel-5.10" = "ec2-user" + "amzn2-ami-hvm-*-x86_64-ebs" = "ec2-user" "al2023-ami-2023" = "ec2-user" "RHEL-7" = "ec2-user" "RHEL-8" = "ec2-user" "RHEL-9" = "ec2-user" } + apm_port = "8200" conf_path = "/tmp/local-apm-config.yml" + bin_path = "/tmp/apm-server" } data "aws_ami" "os" { @@ -101,10 +107,24 @@ data "aws_ami" "os" { owners = [local.image_owners[var.aws_os]] } +data "aws_region" "current" {} + +data "aws_subnets" "public_subnets" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } + filter { + name = "availability-zone" + values = ["${data.aws_region.current.name}a"] + } +} + resource "aws_security_group" "main" { + vpc_id = var.vpc_id egress = [ { - cidr_blocks = ["0.0.0.0/0", ] + cidr_blocks = ["0.0.0.0/0"] description = "" from_port = 0 ipv6_cidr_blocks = [] @@ -117,7 +137,7 @@ resource "aws_security_group" "main" { ] ingress = [ { - cidr_blocks = ["0.0.0.0/0", ] + cidr_blocks = ["0.0.0.0/0"] description = "" from_port = 22 ipv6_cidr_blocks = [] @@ -128,7 +148,7 @@ resource "aws_security_group" "main" { to_port = 22 }, { - cidr_blocks = ["0.0.0.0/0", ] + cidr_blocks = ["0.0.0.0/0"] description = "" from_port = local.apm_port ipv6_cidr_blocks = [] @@ -142,9 +162,12 @@ resource "aws_security_group" "main" { } resource "aws_instance" "apm" { - ami = data.aws_ami.os.id - instance_type = local.instance_types[var.aws_os] - key_name = aws_key_pair.provisioner_key.key_name + ami = data.aws_ami.os.id + instance_type = var.apm_instance_type == "" ? local.instance_types[var.aws_os] : var.apm_instance_type + subnet_id = data.aws_subnets.public_subnets.ids[0] + vpc_security_group_ids = [aws_security_group.main.id] + key_name = aws_key_pair.provisioner_key.key_name + monitoring = false connection { type = "ssh" @@ -153,6 +176,12 @@ resource "aws_instance" "apm" { private_key = file("${var.aws_provisioner_key_name}") } + provisioner "file" { + source = "${var.apm_server_bin_path}/apm-server" + destination = local.bin_path + on_failure = continue + } + provisioner "file" { destination = local.conf_path content = templatefile(var.ea_managed ? "${path.module}/elastic-agent.yml.tftpl" : "${path.module}/apm-server.yml.tftpl", { @@ -172,15 +201,24 @@ resource "aws_instance" "apm" { "sudo cp ${local.conf_path} /etc/elastic-agent/elastic-agent.yml", "sudo systemctl start elastic-agent", "sleep 1", - ] : [ - local.instance_standalone_provision_cmd[var.aws_os], - "sudo cp ${local.conf_path} /etc/apm-server/apm-server.yml", - "sudo systemctl start apm-server", - "sleep 1", - ] + ] : ( + var.apm_server_bin_path == "" ? [ + local.instance_standalone_provision_cmd[var.aws_os], + "sudo cp ${local.conf_path} /etc/apm-server/apm-server.yml", + "sudo systemctl start apm-server", + "sleep 1", + ] : [ + "sudo cp ${local.bin_path} apm-server", + "sudo chmod +x apm-server", + "sudo cp ${local.conf_path} apm-server.yml", + "sudo mkdir -m 777 /var/log/apm-server", + "screen -d -m ./apm-server", + "sleep 1" + ] + ) } - vpc_security_group_ids = [aws_security_group.main.id] + tags = var.tags } resource "null_resource" "apm_server_log" { @@ -208,8 +246,8 @@ data "external" "latest_apm_server" { } resource "aws_key_pair" "provisioner_key" { - key_name = var.aws_provisioner_key_name public_key = file("${var.aws_provisioner_key_name}.pub") + tags = var.tags } resource "random_password" "apm_secret_token" { diff --git a/testing/infra/terraform/modules/standalone_apm_server/outputs.tf b/testing/infra/terraform/modules/standalone_apm_server/outputs.tf index f3cda741079..138e496e592 100644 --- a/testing/infra/terraform/modules/standalone_apm_server/outputs.tf +++ b/testing/infra/terraform/modules/standalone_apm_server/outputs.tf @@ -5,6 +5,11 @@ output "apm_secret_token" { } output "apm_server_url" { - value = "${aws_instance.apm.public_ip}:${local.apm_port}" + value = "http://${aws_instance.apm.public_ip}:${local.apm_port}" description = "The APM Server URL" } + +output "apm_server_ip" { + value = aws_instance.apm.public_ip + description = "The APM Server EC2 IP address" +} diff --git a/testing/infra/terraform/modules/standalone_apm_server/provider.tf b/testing/infra/terraform/modules/standalone_apm_server/provider.tf deleted file mode 100644 index 3d860298151..00000000000 --- a/testing/infra/terraform/modules/standalone_apm_server/provider.tf +++ /dev/null @@ -1,6 +0,0 @@ -provider "aws" { - region = var.worker_region - default_tags { - tags = var.tags - } -} diff --git a/testing/infra/terraform/modules/standalone_apm_server/variables.tf b/testing/infra/terraform/modules/standalone_apm_server/variables.tf index d0e11890832..dbdf8dd7239 100644 --- a/testing/infra/terraform/modules/standalone_apm_server/variables.tf +++ b/testing/infra/terraform/modules/standalone_apm_server/variables.tf @@ -1,12 +1,22 @@ variable "aws_os" { default = "" - description = "Optional aws ec2 instance OS" + description = "Optional aws EC2 instance OS" type = string } -variable "aws_provisioner_key_name" { +variable "apm_instance_type" { default = "" - description = "Optional ssh key name to create the aws key pair and remote provision the ec2 instance" + type = string + description = "Optional apm server instance type overide" +} + +variable "vpc_id" { + description = "VPC ID to provision the EC2 instance" + type = string +} + +variable "aws_provisioner_key_name" { + description = "ssh key name to create the aws key pair and remote provision the EC2 instance" type = string } @@ -33,50 +43,20 @@ variable "stack_version" { type = string } -variable "region" { - default = "gcp-us-west2" - description = "Optional ESS region where to run the smoke tests" - type = string -} - -variable "worker_region" { - default = "us-west-2" - description = "Optional AWS region where the workers will be created. Defaults to us-west-2 (AWS)" - type = string -} - variable "ea_managed" { default = false description = "Whether or not install Elastic Agent managed APM Server" type = bool } +variable "apm_server_bin_path" { + default = "" + type = string + description = "Optionally use the apm-server binary from the specified path instead" +} + variable "tags" { type = map(string) default = {} description = "Optional set of tags to use for all deployments" } - -# CI variables -variable "BRANCH" { - description = "Branch name or pull request for tagging purposes" - default = "unknown" -} - -variable "BUILD_ID" { - description = "Build ID in the CI for tagging purposes" - default = "unknown" -} - -variable "CREATED_DATE" { - description = "Creation date in epoch time for tagging purposes" - default = "unknown" -} - -variable "ENVIRONMENT" { - default = "unknown" -} - -variable "REPO" { - default = "unknown" -} diff --git a/testing/infra/terraform/modules/tags/vars.tf b/testing/infra/terraform/modules/tags/vars.tf index b4ca5f35985..fb84b7d5722 100644 --- a/testing/infra/terraform/modules/tags/vars.tf +++ b/testing/infra/terraform/modules/tags/vars.tf @@ -1,8 +1,10 @@ variable "project" { description = "The value to use for the project tag/label" + type = string } variable "build" { description = "The value to use for the build tag/label" + type = string default = "unknown" } diff --git a/testing/smoke/managed/main.tf b/testing/smoke/managed/main.tf index 3fead67e50e..caf816aa627 100644 --- a/testing/smoke/managed/main.tf +++ b/testing/smoke/managed/main.tf @@ -25,6 +25,10 @@ locals { } } +data "aws_vpc" "default" { + default = true +} + module "ec_deployment" { source = "../../infra/terraform/modules/ec_deployment" region = var.region @@ -44,7 +48,7 @@ module "ec_deployment" { module "standalone_apm_server" { source = "../../infra/terraform/modules/standalone_apm_server" - worker_region = var.worker_region + vpc_id = data.aws_vpc.default.id aws_os = var.aws_os aws_provisioner_key_name = var.aws_provisioner_key_name @@ -52,16 +56,9 @@ module "standalone_apm_server" { elasticsearch_username = module.ec_deployment.elasticsearch_username elasticsearch_password = module.ec_deployment.elasticsearch_password stack_version = var.stack_version - region = var.region tags = merge(local.ci_tags, module.tags.tags) ea_managed = true - - BRANCH = var.BRANCH - BUILD_ID = var.BUILD_ID - CREATED_DATE = var.CREATED_DATE - ENVIRONMENT = var.ENVIRONMENT - REPO = var.REPO } variable "aws_os" { @@ -88,12 +85,6 @@ variable "region" { type = string } -variable "worker_region" { - default = "us-west-2" - description = "Optional AWS region where the workers will be created. Defaults to us-west-2 (AWS)" - type = string -} - # CI variables variable "BRANCH" { description = "Branch name or pull request for tagging purposes" diff --git a/testing/smoke/supported-os/main.tf b/testing/smoke/supported-os/main.tf index 9674b1829ba..9835da198c2 100644 --- a/testing/smoke/supported-os/main.tf +++ b/testing/smoke/supported-os/main.tf @@ -25,6 +25,10 @@ locals { } } +data "aws_vpc" "default" { + default = true +} + module "ec_deployment" { source = "../../infra/terraform/modules/ec_deployment" region = var.region @@ -44,7 +48,7 @@ module "ec_deployment" { module "standalone_apm_server" { source = "../../infra/terraform/modules/standalone_apm_server" - worker_region = var.worker_region + vpc_id = data.aws_vpc.default.id aws_os = var.aws_os aws_provisioner_key_name = var.aws_provisioner_key_name @@ -52,16 +56,9 @@ module "standalone_apm_server" { elasticsearch_username = module.ec_deployment.elasticsearch_username elasticsearch_password = module.ec_deployment.elasticsearch_password stack_version = var.stack_version - region = var.region tags = merge(local.ci_tags, module.tags.tags) ea_managed = false - - BRANCH = var.BRANCH - BUILD_ID = var.BUILD_ID - CREATED_DATE = var.CREATED_DATE - ENVIRONMENT = var.ENVIRONMENT - REPO = var.REPO } variable "aws_os" { @@ -88,12 +85,6 @@ variable "region" { type = string } -variable "worker_region" { - default = "us-west-2" - description = "Optional AWS region where the workers will be created. Defaults to us-west-2 (AWS)" - type = string -} - # CI variables variable "BRANCH" { description = "Branch name or pull request for tagging purposes" diff --git a/tools/go.mod b/tools/go.mod index 6c3fc8c4cec..8b7a7226c63 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -3,6 +3,7 @@ module github.com/elastic/apm-server/tools go 1.23.0 require ( + github.com/elastic/apm-perf v0.0.0-20240925232339-499ba2a27fd4 github.com/elastic/apm-tools v0.0.0-20230828065051-3f799314cc8b github.com/elastic/go-licenser v0.4.2 github.com/elastic/gobench v0.0.0-20220608141032-f30bc57e329c @@ -75,7 +76,7 @@ require ( github.com/invopop/jsonschema v0.12.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.10 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -111,9 +112,9 @@ require ( github.com/spf13/viper v1.19.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/terraform-docs/terraform-config-inspect v0.0.0-20210728164355-9c1f178932fa // indirect - github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect @@ -121,6 +122,7 @@ require ( github.com/zclconf/go-cty v1.15.0 // indirect gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect @@ -133,7 +135,7 @@ require ( golang.org/x/tools v0.25.0 // indirect golang.org/x/tools/go/vcs v0.1.0-deprecated // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.66.2 // indirect + google.golang.org/grpc v1.67.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/tools/go.sum b/tools/go.sum index 5069f0920c1..837f563f5b8 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -69,6 +69,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/elastic/apm-perf v0.0.0-20240925232339-499ba2a27fd4 h1:IMvFDtKRa3QSkj1Zr0yesEwubOCp9YARpR4KrHOMYb0= +github.com/elastic/apm-perf v0.0.0-20240925232339-499ba2a27fd4/go.mod h1:K/l3iP9/AYOQ5QTb0JZbWR+Law4vLfpLFmhZ+5KoRRY= github.com/elastic/apm-tools v0.0.0-20230828065051-3f799314cc8b h1:47VFNMVQQt7WTHF7NdS37Qkk4fMx4uGhM5WWf5lUqio= github.com/elastic/apm-tools v0.0.0-20230828065051-3f799314cc8b/go.mod h1:IUMdGldUOSqfeXSj+zIjYaBX6WFM7M95WU2VdmZ5ZyE= github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= @@ -170,8 +172,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= +github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -292,12 +294,13 @@ github.com/terraform-docs/terraform-config-inspect v0.0.0-20210728164355-9c1f178 github.com/terraform-docs/terraform-docs v0.19.0 h1:YDC9YTxEzH0NPbXaqSwi1opp3+F2Ka3ZL/vIUa1Dv68= github.com/terraform-docs/terraform-docs v0.19.0/go.mod h1:+Hd2ngjXmsC9H9hHDz4BXdxLaOHJX3/cZEWqaeaG8m0= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= @@ -319,8 +322,12 @@ gitlab.com/digitalxero/go-conventional-commit v1.0.7 h1:8/dO6WWG+98PMhlZowt/Yjui gitlab.com/digitalxero/go-conventional-commit v1.0.7/go.mod h1:05Xc2BFsSyC5tKhK0y+P3bs0AwUtNuTp+mTpbCU/DZ0= go.elastic.co/go-licence-detector v0.7.0 h1:qC31sfyfNcNx/zMYcLABU0ac3MbGHZgksCAb5lMDUMg= go.elastic.co/go-licence-detector v0.7.0/go.mod h1:f5ty8pjynzQD8BcS+s0qtlOGKc35/HKQxCVi8SHhV5k= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -448,8 +455,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= +google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/tools.go b/tools/tools.go index f869641019c..4484e8a01f1 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -38,4 +38,6 @@ import ( _ "github.com/elastic/go-licenser" // go.mod/go.sum _ "github.com/elastic/gobench" // go.mod/go.sum + + _ "github.com/elastic/apm-perf/cmd/moxy" // go.mod/go.sum ) diff --git a/x-pack/apm-server/main.go b/x-pack/apm-server/main.go index 2ec2c6d2c0b..893681c966a 100644 --- a/x-pack/apm-server/main.go +++ b/x-pack/apm-server/main.go @@ -12,7 +12,7 @@ import ( "sync" "github.com/dgraph-io/badger/v2" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "golang.org/x/sync/errgroup" "github.com/elastic/beats/v7/libbeat/common/reload" @@ -275,7 +275,6 @@ func cleanup() error { } func Main() error { - beatcmd.InitBeatCmd() rootCmd := newXPackRootCommand( func(args beatcmd.RunnerParams) (beatcmd.Runner, error) { return beater.NewRunner(beater.RunnerParams{ diff --git a/x-pack/apm-server/root_test.go b/x-pack/apm-server/root_test.go index d916a0e73f8..39fbf2aa30d 100644 --- a/x-pack/apm-server/root_test.go +++ b/x-pack/apm-server/root_test.go @@ -13,7 +13,6 @@ import ( ) func TestSubCommands(t *testing.T) { - beatcmd.InitBeatCmd() rootCmd := newXPackRootCommand(func(beatcmd.RunnerParams) (beatcmd.Runner, error) { panic("unexpected call") }) diff --git a/x-pack/apm-server/sampling/eventstorage/sharded_bench_test.go b/x-pack/apm-server/sampling/eventstorage/sharded_bench_test.go index 7ef76309093..1d26986eed6 100644 --- a/x-pack/apm-server/sampling/eventstorage/sharded_bench_test.go +++ b/x-pack/apm-server/sampling/eventstorage/sharded_bench_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/elastic/apm-data/model/modelpb" "github.com/elastic/apm-server/x-pack/apm-server/sampling/eventstorage" diff --git a/x-pack/apm-server/sampling/eventstorage/storage_bench_test.go b/x-pack/apm-server/sampling/eventstorage/storage_bench_test.go index 90f58fbd268..3c5bafcf17a 100644 --- a/x-pack/apm-server/sampling/eventstorage/storage_bench_test.go +++ b/x-pack/apm-server/sampling/eventstorage/storage_bench_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "github.com/elastic/apm-data/model/modelpb" diff --git a/x-pack/apm-server/sampling/eventstorage/storage_test.go b/x-pack/apm-server/sampling/eventstorage/storage_test.go index 3f35924965e..e22ae7c7010 100644 --- a/x-pack/apm-server/sampling/eventstorage/storage_test.go +++ b/x-pack/apm-server/sampling/eventstorage/storage_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/dgraph-io/badger/v2" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" diff --git a/x-pack/apm-server/sampling/eventstorage/storage_whitebox_test.go b/x-pack/apm-server/sampling/eventstorage/storage_whitebox_test.go index 4dbc4cc0b02..7abae0980ed 100644 --- a/x-pack/apm-server/sampling/eventstorage/storage_whitebox_test.go +++ b/x-pack/apm-server/sampling/eventstorage/storage_whitebox_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/dgraph-io/badger/v2" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/x-pack/apm-server/sampling/groups_test.go b/x-pack/apm-server/sampling/groups_test.go index 1312f806e39..10fa7e87322 100644 --- a/x-pack/apm-server/sampling/groups_test.go +++ b/x-pack/apm-server/sampling/groups_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/x-pack/apm-server/sampling/processor_test.go b/x-pack/apm-server/sampling/processor_test.go index 6b663f3262a..f17500da501 100644 --- a/x-pack/apm-server/sampling/processor_test.go +++ b/x-pack/apm-server/sampling/processor_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/stretchr/testify/assert" diff --git a/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go b/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go index f27eeecb458..7927a279776 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" diff --git a/x-pack/apm-server/sampling/pubsub/pubsub_test.go b/x-pack/apm-server/sampling/pubsub/pubsub_test.go index e7e52086d8d..5bb73f6b538 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub_test.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/gofrs/uuid/v5" + "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup"