[7.17] feat(slack): apm-server (backport #13307) (#13344)

* feat(slack): apm-server (#13307) * feat(slack): apm-server * fix name * Apply suggestions from code review (cherry picked from commit 055c3a5) # Conflicts: # .github/workflows/benchmarks.yml # .github/workflows/run-minor-release.yml # .github/workflows/run-patch-release.yml # .github/workflows/smoke-tests-schedule.yml * Apply suggestions from code review * Apply suggestions from code review --------- Co-authored-by: Victor Martinez <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
elastic · Jun 9, 2024 · 8f3028f · 8f3028f
1 parent 7b3231c
commit 8f3028f
Show file tree

Hide file tree

Showing 4 changed files with 318 additions and 40 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -0,0 +1,233 @@
+name: benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      profile:
+        description: 'The system profile used to run the benchmarks'
+        required: false
+        type: string
+      runOnStable:
+        description: 'Run the benchmarks on the latest stable version'
+        required: false
+        type: boolean
+        default: false
+      benchmarkAgents:
+        description: 'Set the number of agents to send data to the APM Server'
+        required: false
+        type: string
+      benchmarkRun:
+        description: 'Set the expression that matches the benchmark scenarios to run'
+        required: false
+        type: string
+  schedule:
+    - cron: '0 17 * * *'
+
+env:
+  PNG_REPORT_FILE: out.png
+  BENCHMARK_RESULT: benchmark-result.txt
+  WORKING_DIRECTORY: testing/benchmark
+
+permissions:
+  contents: read
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+    env:
+      SSH_KEY: ./id_rsa_terraform
+      TF_VAR_private_key: ./id_rsa_terraform
+      TF_VAR_public_key: ./id_rsa_terraform.pub
+      TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
+      TF_VAR_BUILD_ID: ${{ github.run_id }}
+      TF_VAR_ENVIRONMENT: ci
+      TF_VAR_REPO: ${{ github.repository }}
+      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - uses: rlespinasse/github-slug-action@797d68864753cbceedc271349d402da4590e6302
+
+      - name: Set up env
+        run: |
+          SLUGGED_BRANCH_NAME=${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+          CREATED_AT=$(date +%s)
+          echo "TF_VAR_BRANCH=${SLUGGED_BRANCH_NAME}" >> "$GITHUB_ENV"
+          echo "TF_VAR_CREATED_AT=${CREATED_AT}" >> "$GITHUB_ENV"
+          echo "USER=benchci-$SLUGGED_BRANCH_NAME-$CREATED_AT" >> "$GITHUB_ENV"
+
+          if [ ! -z "${{ inputs.benchmarkAgents }}" ]; then
+            echo "BENCHMARK_AGENTS=${{ inputs.benchmarkAgents }}" >> "$GITHUB_ENV"
+          fi
+          if [ ! -z "${{ inputs.benchmarkRun }}" ]; then
+            echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV"
+          fi
+
+      - uses: hashicorp/[email protected]
+        env:
+          AWS_CREDENTIALS_PATH: secret/observability-team/ci/elastic-observability-aws-account-auth
+          BENCHMARK_CLOUD_CREDENTIALS_PATH: secret/observability-team/ci/benchmark-cloud
+          EC_CREDENTIALS_PATH: secret/observability-team/ci/elastic-cloud/observability-team-pro
+          KIBANA_CREDENTIALS_PATH: secret/observability-team/ci/apm-benchmark-kibana
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          method: approle
+          secrets: |
+            ${{ env.AWS_CREDENTIALS_PATH }} access_key | AWS_ACCESS_KEY_ID ;
+            ${{ env.AWS_CREDENTIALS_PATH }} secret_key | AWS_SECRET_ACCESS_KEY ;
+            ${{ env.BENCHMARK_CLOUD_CREDENTIALS_PATH }} user | GOBENCH_USERNAME ;
+            ${{ env.BENCHMARK_CLOUD_CREDENTIALS_PATH }} password | GOBENCH_PASSWORD ;
+            ${{ env.BENCHMARK_CLOUD_CREDENTIALS_PATH }} url | GOBENCH_HOST ;
+            ${{ env.EC_CREDENTIALS_PATH }} apiKey | EC_API_KEY ;
+            ${{ env.KIBANA_CREDENTIALS_PATH }} user | KIBANA_USERNAME ;
+            ${{ env.KIBANA_CREDENTIALS_PATH }} password | KIBANA_PASSWORD ;
+            ${{ env.KIBANA_CREDENTIALS_PATH }} kibana_url | KIBANA_ENDPOINT ;
+            ${{ env.KIBANA_CREDENTIALS_PATH }} kibana_dashboard_url | KIBANA_DASHBOARD_URL ;
+
+      - name: Log in to the Elastic Container registry
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        with:
+          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
+          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
+          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
+
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.3.7
+          terraform_wrapper: false
+
+      - name: Build apmbench
+        run: make apmbench $SSH_KEY terraform.tfvars
+
+      - name: Override docker committed version
+        if: ${{ ! inputs.runOnStable }}
+        run: make docker-override-committed-version
+
+      - name: Spin up benchmark environment
+        id: deploy
+        run: |
+          make init apply
+          admin_console_url=$(terraform output -raw admin_console_url)
+          echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT"
+          echo "-> infra setup done"
+
+      - name: Run benchmarks autotuned
+        if: ${{ inputs.benchmarkAgents == '' }}
+        run: make run-benchmark-autotuned index-benchmark-results
+
+      - name: Run benchmarks self tuned
+        if: ${{ inputs.benchmarkAgents != '' }}
+        run: make run-benchmark index-benchmark-results
+
+      - name: Download PNG
+        run: >-
+          ${{ github.workspace }}/.ci/scripts/download-png-from-kibana.sh
+          $KIBANA_ENDPOINT
+          $KIBANA_USERNAME
+          $KIBANA_PASSWORD
+          $PNG_REPORT_FILE
+
+      - name: Upload PNG
+        uses: actions/upload-artifact@v4
+        with:
+          name: kibana-png-report
+          path: ${{ env.WORKING_DIRECTORY }}/${{ env.PNG_REPORT_FILE }}
+          if-no-files-found: error
+
+      - name: Upload PNG to AWS S3
+        id: s3-upload-png
+        env:
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          DEST_NAME="github-run-id-${{ github.run_id }}.png"
+          aws s3 --debug cp ${{ env.PNG_REPORT_FILE }} s3://elastic-apm-server-benchmark-reports/${DEST_NAME}
+          echo "png_report_url=https://elastic-apm-server-benchmark-reports.s3.amazonaws.com/${DEST_NAME}" >> "$GITHUB_OUTPUT"
+
+      - name: Upload benchmark result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: benchmark-result
+          path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_RESULT }}
+          if-no-files-found: error
+
+      - name: Tear down benchmark environment
+        if: always()
+        run: make destroy
+
+      # Notify failure to Slack only on schedule (nightly run)
+      - if: failure() && github.event_name == 'schedule'
+        uses: elastic/oblt-actions/slack/[email protected]
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#apm-server"
+          message: |
+            Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!
+
+      # Notify result to Slack only on schedule (nightly run)
+      - if: github.event_name == 'schedule'
+        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        with:
+          channel-id: "#apm-server"
+          payload: |
+            {
+                "blocks": [
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!"
+                        },
+                        "accessory": {
+                            "type": "button",
+                            "style": "primary",
+                            "text": {
+                                "type": "plain_text",
+                                "text": "Workflow Run #${{ github.run_id }}",
+                                "emoji": true
+                            },
+                            "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                            "action_id": "workflow-run-button"
+                        }
+                    },
+                    {
+                        "type": "image",
+                        "image_url": "${{ steps.s3-upload-png.outputs.png_report_url }}",
+                        "alt_text": "kibana-png-report"
+                    },
+                    {
+                        "type": "actions",
+                        "elements": [
+                            {
+                                "type": "button",
+                                "text": {
+                                    "type": "plain_text",
+                                    "text": "Benchmarks dashboard"
+                                },
+                                "url": "${{ env.KIBANA_DASHBOARD_URL }}",
+                                "action_id": "kibana-dashboard-button"
+                            },
+                            {
+                                "type": "button",
+                                "text": {
+                                    "type": "plain_text",
+                                    "text": "Elastic Cloud deployment"
+                                },
+                                "url": "${{ steps.deploy.outputs.admin_console_url }}",
+                                "action_id": "admin-console-button"
+                            }
+                        ]
+                    }
+                ]
+            }
diff --git a/.github/workflows/prepare-release/action.yml b/.github/workflows/prepare-release/action.yml
@@ -3,21 +3,15 @@ name: prepare-release
 description: Common tasks for preparing minor and patch releases
 
 inputs:
+  slack-bot-token:
+    description: 'The slack bot token'
+    required: true
   type:
     description: 'Release type (minor or patch)'
     required: true
   version:
     description: 'The version'
     required: true
-  vault-url:
-    description: 'Vault URL'
-    required: true
-  vault-role-id:
-    description: 'Vault role ID'
-    required: true
-  vault-secret-id:
-    description: 'Vault secret ID'
-    required: true
 
 outputs:
   release-branch:
@@ -28,19 +22,17 @@ outputs:
     value: ${{ steps.generate.outputs.release-version }}
   slack-thread:
     description: "Slack thread id"
-    value: ${{ steps.slack-thread.outputs.threadTimestamp }}
+    value: ${{ steps.slack-thread.outputs.thread-timestamp }}
 
 runs:
   using: "composite"
   steps:
     - name: Send slack message when started
       id: slack-thread
-      uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
+      uses: elastic/oblt-actions/slack/[email protected]
       with:
-        url: ${{ inputs.vault-url }}
-        roleId: ${{ inputs.vault-role-id }}
-        secretId: ${{ inputs.vault-secret-id }}
-        channel: ${{ env.SLACK_CHANNEL }}
+        bot-token: ${{ inputs.slack-bot-token }}
+        channel-id: ${{ env.SLACK_CHANNEL }}
         message: ":wave: This is the thread for the ${{ inputs.type }} release `${{ github.repository }}@${{ env.VERSION }}`. (<${{ env.JOB_URL }}|workflow run>)"
       env:
         VERSION: ${{ inputs.version }}
@@ -93,17 +85,15 @@ runs:
       env:
         TAG: 'refs/tags/v${{ steps.generate.outputs.release-version }}'
 
-    - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
-      if: failure()
+    - if: failure()
+      uses: elastic/oblt-actions/slack/[email protected]
       with:
-        url: ${{ inputs.vault-url }}
-        roleId: ${{ inputs.vault-role-id }}
-        secretId: ${{ inputs.vault-secret-id }}
-        channel: ${{ env.SLACK_CHANNEL }}
-        threadTimestamp: ${{ steps.slack-thread.outputs.threadTimestamp || '' }}
+        bot-token: ${{ inputs.slack-bot-token }}
+        channel-id: ${{ env.SLACK_CHANNEL }}
         message: |-
           :fire: Something went wrong with the ${{ inputs.type }} release preparation. It failed with the below error message:
           `${{ env.FAILURE_MESSAGE }}`.
           See <${{ env.JOB_URL }}|logs>.
+        thread-timestamp: ${{ steps.slack-thread.outputs.thread-timestamp || '' }}
       env:
         JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
diff --git a/.github/workflows/run-patch-release.yml b/.github/workflows/run-patch-release.yml
@@ -34,9 +34,7 @@ jobs:
       - id: prepare
         uses: ./.github/workflows/prepare-release
         with:
-          vault-url: ${{ secrets.VAULT_ADDR }}
-          vault-role-id: ${{ secrets.VAULT_ROLE_ID }}
-          vault-secret-id: ${{ secrets.VAULT_SECRET_ID }}
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           version: ${{ inputs.version }}
           type: 'patch'
 
@@ -72,25 +70,21 @@ jobs:
 
       - run: make patch-release
 
-      - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
-        if: success()
+      - if: success()
+        uses: elastic/oblt-actions/slack/[email protected]
         with:
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
-          channel: ${{ env.SLACK_CHANNEL }}
-          threadTimestamp: ${{ needs.prepare.outputs.slack-thread || '' }}
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: ${{ env.SLACK_CHANNEL }}
           message: |-
             Feature freeze for `${{ github.repository }}@${{ env.RELEASE_VERSION }}` is Today.
             All the relevant PRs and issues have been created.
+          thread-timestamp: ${{ needs.prepare.outputs.slack-thread || '' }}
 
-      - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
-        if: failure()
+      - if: failure()
+        uses: elastic/oblt-actions/slack/[email protected]
         with:
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
-          channel: ${{ env.SLACK_CHANNEL }}
-          threadTimestamp: ${{ needs.prepare.outputs.slack-thread || '' }}
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: ${{ env.SLACK_CHANNEL }}
           message: |-
-            :fire: Something went wrong with the patch release. See <${{ env.JOB_URL }}|logs>.
+            :fire: Something went wrong with the patch release. See <${{ env.JOB_URL }}|logs>.
+          thread-timestamp: ${{ needs.prepare.outputs.slack-thread || '' }}