From a2c42ea7cf385183702bed789624897c68359513 Mon Sep 17 00:00:00 2001 From: Kostiantyn Masliuk <1pkg@protonmail.com> Date: Thu, 22 Aug 2024 15:00:36 -0700 Subject: [PATCH] Finalize PGO benchmark pipeline update. --- .github/workflows/benchmarks.yml | 340 +++++++++++++++---------------- 1 file changed, 169 insertions(+), 171 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 78181f2f99f..0a563e2c59a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -30,8 +30,7 @@ env: WORKING_DIRECTORY: testing/benchmark permissions: - contents: write - pull-requests: write + contents: read jobs: benchmarks: @@ -58,129 +57,129 @@ jobs: steps: - uses: actions/checkout@v4 - # - uses: actions/setup-go@v5 - # with: - # go-version-file: 'go.mod' + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' - # - uses: rlespinasse/github-slug-action@797d68864753cbceedc271349d402da4590e6302 + - uses: rlespinasse/github-slug-action@797d68864753cbceedc271349d402da4590e6302 - # - name: Set up env - # run: | - # SLUGGED_BRANCH_NAME=${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} - # CREATED_AT=$(date +%s) - # echo "TF_VAR_BRANCH=${SLUGGED_BRANCH_NAME}" >> "$GITHUB_ENV" - # echo "TF_VAR_CREATED_AT=${CREATED_AT}" >> "$GITHUB_ENV" - # echo "USER=benchci-$SLUGGED_BRANCH_NAME-$CREATED_AT" >> "$GITHUB_ENV" + - name: Set up env + run: | + SLUGGED_BRANCH_NAME=${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} + CREATED_AT=$(date +%s) + echo "TF_VAR_BRANCH=${SLUGGED_BRANCH_NAME}" >> "$GITHUB_ENV" + echo "TF_VAR_CREATED_AT=${CREATED_AT}" >> "$GITHUB_ENV" + echo "USER=benchci-$SLUGGED_BRANCH_NAME-$CREATED_AT" >> "$GITHUB_ENV" - # if [ ! -z "${{ inputs.benchmarkAgents }}" ]; then - # echo "BENCHMARK_AGENTS=${{ inputs.benchmarkAgents }}" >> "$GITHUB_ENV" - # fi - # if [ ! -z "${{ inputs.benchmarkRun }}" ]; then - # echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV" - # fi + if [ ! -z "${{ inputs.benchmarkAgents }}" ]; then + echo "BENCHMARK_AGENTS=${{ inputs.benchmarkAgents }}" >> "$GITHUB_ENV" + fi + if [ ! -z "${{ inputs.benchmarkRun }}" ]; then + echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV" + fi - # - name: Log in to the Elastic Container registry - # uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - # with: - # registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} - # username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} - # password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} + - name: Log in to the Elastic Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} + username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} + password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} - # - uses: elastic/oblt-actions/google/auth@v1 + - uses: elastic/oblt-actions/google/auth@v1 - # - uses: elastic/oblt-actions/aws/auth@v1 - # with: - # role-duration-seconds: 18000 # 5 hours + - uses: elastic/oblt-actions/aws/auth@v1 + with: + role-duration-seconds: 18000 # 5 hours - # - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4 - # with: - # export_to_environment: true - # secrets: |- - # EC_API_KEY:elastic-observability/elastic-cloud-observability-team-pro-api-key + - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4 + with: + export_to_environment: true + secrets: |- + EC_API_KEY:elastic-observability/elastic-cloud-observability-team-pro-api-key - # - uses: hashicorp/setup-terraform@v3 - # with: - # terraform_version: 1.3.7 - # terraform_wrapper: false + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.3.7 + terraform_wrapper: false - # - name: Build apmbench - # run: make apmbench $SSH_KEY terraform.tfvars + - name: Build apmbench + run: make apmbench $SSH_KEY terraform.tfvars - # - name: Override docker committed version - # if: ${{ ! inputs.runOnStable }} - # run: make docker-override-committed-version + - name: Override docker committed version + if: ${{ ! inputs.runOnStable }} + run: make docker-override-committed-version - # - name: Spin up benchmark environment - # id: deploy - # run: | - # make init apply - # admin_console_url=$(terraform output -raw admin_console_url) - # echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT" - # echo "-> infra setup done" + - name: Spin up benchmark environment + id: deploy + run: | + make init apply + admin_console_url=$(terraform output -raw admin_console_url) + echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT" + echo "-> infra setup done" - # - name: Run benchmarks autotuned - # if: ${{ inputs.benchmarkAgents == '' }} - # run: make run-benchmark-autotuned + - name: Run benchmarks autotuned + if: ${{ inputs.benchmarkAgents == '' }} + run: make run-benchmark-autotuned - # - name: Run benchmarks self tuned - # if: ${{ inputs.benchmarkAgents != '' }} - # run: make run-benchmark + - name: Run benchmarks self tuned + if: ${{ inputs.benchmarkAgents != '' }} + run: make run-benchmark - # # Results are only indexed and uploaded if the run happens on the main branch. + # Results are only indexed and uploaded if the run happens on the main branch. - # - name: Index benchmarks result - # if: github.ref == 'refs/heads/main' - # run: make index-benchmark-results + - name: Index benchmarks result + if: github.ref == 'refs/heads/main' + run: make index-benchmark-results - # - name: Download PNG - # if: github.ref == 'refs/heads/main' - # run: >- - # ${{ github.workspace }}/.ci/scripts/download-png-from-kibana.sh - # ${{ secrets.KIBANA_BENCH_ENDPOINT }} - # ${{ secrets.KIBANA_BENCH_USERNAME }} - # ${{ secrets.KIBANA_BENCH_PASSWORD }} - # $PNG_REPORT_FILE + - name: Download PNG + if: github.ref == 'refs/heads/main' + run: >- + ${{ github.workspace }}/.ci/scripts/download-png-from-kibana.sh + ${{ secrets.KIBANA_BENCH_ENDPOINT }} + ${{ secrets.KIBANA_BENCH_USERNAME }} + ${{ secrets.KIBANA_BENCH_PASSWORD }} + $PNG_REPORT_FILE - # - name: Upload PNG - # if: github.ref == 'refs/heads/main' - # uses: actions/upload-artifact@v4 - # with: - # name: kibana-png-report - # path: ${{ env.WORKING_DIRECTORY }}/${{ env.PNG_REPORT_FILE }} - # if-no-files-found: error + - name: Upload PNG + if: github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: kibana-png-report + path: ${{ env.WORKING_DIRECTORY }}/${{ env.PNG_REPORT_FILE }} + if-no-files-found: error - # - name: Upload PNG to AWS S3 - # if: github.ref == 'refs/heads/main' - # id: s3-upload-png - # env: - # AWS_DEFAULT_REGION: us-east-1 - # run: | - # DEST_NAME="github-run-id-${{ github.run_id }}.png" - # aws s3 --debug cp ${{ env.PNG_REPORT_FILE }} s3://elastic-apm-server-benchmark-reports/${DEST_NAME} - # echo "png_report_url=https://elastic-apm-server-benchmark-reports.s3.amazonaws.com/${DEST_NAME}" >> "$GITHUB_OUTPUT" + - name: Upload PNG to AWS S3 + if: github.ref == 'refs/heads/main' + id: s3-upload-png + env: + AWS_DEFAULT_REGION: us-east-1 + run: | + DEST_NAME="github-run-id-${{ github.run_id }}.png" + aws s3 --debug cp ${{ env.PNG_REPORT_FILE }} s3://elastic-apm-server-benchmark-reports/${DEST_NAME} + echo "png_report_url=https://elastic-apm-server-benchmark-reports.s3.amazonaws.com/${DEST_NAME}" >> "$GITHUB_OUTPUT" - # - name: Upload benchmark result - # if: github.ref == 'refs/heads/main' - # uses: actions/upload-artifact@v4 - # with: - # name: benchmark-result - # path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_RESULT }} - # if-no-files-found: error + - name: Upload benchmark result + if: github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: benchmark-result + path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_RESULT }} + if-no-files-found: error - # # The next section injects CPU profile collected by apmbench into the build. - # # By copying the profile, uploading it to the artifacts and pushing it - # # via a PR to update default.pgo. + # The next section injects CPU profile collected by apmbench into the build. + # By copying the profile, uploading it to the artifacts and pushing it + # via a PR to update default.pgo. - # - name: Copy CPU profile - # run: make cp-cpuprof + - name: Copy CPU profile + run: make cp-cpuprof - # - name: Upload CPU profile - # uses: actions/upload-artifact@v4 - # with: - # name: cpu-profile - # path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_CPU_OUT }} - # if-no-files-found: error + - name: Upload CPU profile + uses: actions/upload-artifact@v4 + with: + name: cpu-profile + path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_CPU_OUT }} + if-no-files-found: error - name: Open update PGO PR run: | @@ -191,7 +190,6 @@ jobs: git checkout main BRANCH="update-pgo-$(date +%s)" git checkout -b "$BRANCH" - echo "test" > default.pgo git add default.pgo git commit -m "PGO: Update default.pgo from benchmarks $WORKFLOW." git push -u origin "$BRANCH" @@ -200,73 +198,73 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WORKFLOW: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} - # - name: Tear down benchmark environment - # if: always() - # run: make destroy + - name: Tear down benchmark environment + if: always() + run: make destroy - # # Notify failure to Slack only on schedule (nightly run) - # - if: failure() && github.event_name == 'schedule' - # uses: elastic/oblt-actions/slack/notify-result@v1 - # with: - # bot-token: ${{ secrets.SLACK_BOT_TOKEN }} - # channel-id: "#apm-server" - # message: Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this ! + # Notify failure to Slack only on schedule (nightly run) + - if: failure() && github.event_name == 'schedule' + uses: elastic/oblt-actions/slack/notify-result@v1 + with: + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: "#apm-server" + message: Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this ! - # # Notify result to Slack only on schedule (nightly run) - # - if: github.event_name == 'schedule' - # uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 - # env: - # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - # with: - # channel-id: "#apm-server" - # payload: | - # { - # "blocks": [ - # { - # "type": "section", - # "text": { - # "type": "mrkdwn", - # "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!" - # }, - # "accessory": { - # "type": "button", - # "style": "primary", - # "text": { - # "type": "plain_text", - # "text": "Workflow Run #${{ github.run_id }}", - # "emoji": true - # }, - # "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", - # "action_id": "workflow-run-button" - # } - # }, - # { - # "type": "image", - # "image_url": "${{ steps.s3-upload-png.outputs.png_report_url }}", - # "alt_text": "kibana-png-report" - # }, - # { - # "type": "actions", - # "elements": [ - # { - # "type": "button", - # "text": { - # "type": "plain_text", - # "text": "Benchmarks dashboard" - # }, - # "url": "${{ secrets.KIBANA_BENCH_DASHBOARD }}", - # "action_id": "kibana-dashboard-button" - # }, - # { - # "type": "button", - # "text": { - # "type": "plain_text", - # "text": "Elastic Cloud deployment" - # }, - # "url": "${{ steps.deploy.outputs.admin_console_url }}", - # "action_id": "admin-console-button" - # } - # ] - # } - # ] - # } + # Notify result to Slack only on schedule (nightly run) + - if: github.event_name == 'schedule' + uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + with: + channel-id: "#apm-server" + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!" + }, + "accessory": { + "type": "button", + "style": "primary", + "text": { + "type": "plain_text", + "text": "Workflow Run #${{ github.run_id }}", + "emoji": true + }, + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "action_id": "workflow-run-button" + } + }, + { + "type": "image", + "image_url": "${{ steps.s3-upload-png.outputs.png_report_url }}", + "alt_text": "kibana-png-report" + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": "Benchmarks dashboard" + }, + "url": "${{ secrets.KIBANA_BENCH_DASHBOARD }}", + "action_id": "kibana-dashboard-button" + }, + { + "type": "button", + "text": { + "type": "plain_text", + "text": "Elastic Cloud deployment" + }, + "url": "${{ steps.deploy.outputs.admin_console_url }}", + "action_id": "admin-console-button" + } + ] + } + ] + }