Merge branch 'main' into issue-11282

elastic · Nov 27, 2023 · da7ab68 · da7ab68
2 parents ddcb440 + 52cf775
commit da7ab68
Show file tree

Hide file tree

Showing 209 changed files with 1,746 additions and 2,385 deletions.
diff --git a/.buildkite/package.yml b/.buildkite/package.yml
@@ -1,6 +1,8 @@
 notify:
   - slack: "#apm-server"
     if: build.state == "failed"
+  - slack: "#observablt-bots"
+    if: "build.state != 'passed'"
 
 env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"

diff --git a/.ci/jobs/apm-server-package-mbp.yml b/.ci/jobs/apm-server-package-mbp.yml
@@ -14,7 +14,7 @@
         discover-pr-origin: merge-current
         discover-tags: false
         disable-pr-notifications: true
-        head-filter-regex: '(main|7\.17|8\.\d+)'
+        head-filter-regex: '(main|7\.17|8\.9|8\.1\d+)'
         notification-context: 'apm-server-package'
         repo: apm-server
         repo-owner: elastic

diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml
@@ -3,7 +3,7 @@
 ##### GLOBAL METADATA
 
 - meta:
-    cluster: apm-ci
+    cluster: beats-ci
 
 ##### JOB DEFAULTS
 

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,42 @@
+version: 2
+updates:
+
+  # GitHub actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    reviewers:
+      - "elastic/observablt-ci"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "22:00"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+
+  # GitHub composite actions
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows/generate-smoke-tests-list"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "22:00"
+    reviewers:
+      - "elastic/observablt-ci"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+  - package-ecosystem: "github-actions"
+    directory: "/.github/workflows/setup-cluster-env"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "22:00"
+    reviewers:
+      - "elastic/observablt-ci"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -44,13 +44,13 @@ jobs:
       TF_VAR_REPO: ${{ github.repository }}
       GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
 
-      - uses: rlespinasse/github-slug-action@d1ca8ffbce40ea2371e35bca356db0acacf567d2
+      - uses: rlespinasse/github-slug-action@00d4236bb38fcea6fc4aa6b1dacc7187fc8e38a0
 
       - name: Set up env
         run: |
@@ -67,7 +67,7 @@ jobs:
             echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV"
           fi
 
-      - uses: hashicorp/vault-action@v2.6.0
+      - uses: hashicorp/vault-action@v2.7.4
         env:
           AWS_CREDENTIALS_PATH: secret/observability-team/ci/elastic-observability-aws-account-auth
           BENCHMARK_CLOUD_CREDENTIALS_PATH: secret/observability-team/ci/benchmark-cloud
@@ -97,7 +97,7 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-      - uses: hashicorp/setup-terraform@v2
+      - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.3.7
           terraform_wrapper: false
@@ -166,7 +166,7 @@ jobs:
         uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
         with:
           message: |
-            Nightly benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!
+            Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!
           vaultUrl: ${{ secrets.VAULT_ADDR }}
           vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
           vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
@@ -187,7 +187,7 @@ jobs:
                         "type": "section",
                         "text": {
                             "type": "mrkdwn",
-                            "text": "${{ github.event_name == 'schedule' && 'Nightly succesfully executed!' || 'Benchmarks succesfully executed!' }}"
+                            "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!"
                         },
                         "accessory": {
                             "type": "button",

diff --git a/.github/workflows/bump-elastic-stack.yml b/.github/workflows/bump-elastic-stack.yml
@@ -26,7 +26,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.filter.outputs.matrix) }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
 

diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current
         with:
           vaultUrl: ${{ secrets.VAULT_ADDR }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -26,8 +26,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true
@@ -45,18 +45,20 @@ jobs:
         os: ['macos-latest', 'ubuntu-latest', 'windows-latest']
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true
-      - run: go test -v ./...
+      - env:
+          CGO_ENABLED: "0"
+        run: go test -v ./...
 
   system-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: systemtest/go.mod
           cache: true

diff --git a/.github/workflows/generate-smoke-tests-list/action.yml b/.github/workflows/generate-smoke-tests-list/action.yml
@@ -33,6 +33,7 @@ runs:
     - id: generate
       name: Generate matrix and date
       run: |
-        echo "tests=$(make smoketest/discover)" >> "${GITHUB_OUTPUT}"
+        # remove the full path and use a relative path instead to be github runner agnostic
+        echo "tests=$(make smoketest/discover | sed "s#${GITHUB_WORKSPACE}/#./#g")" >> "${GITHUB_OUTPUT}"
         echo "date=$(date +%s)" >> "${GITHUB_OUTPUT}"
       shell: 'bash'
diff --git a/.github/workflows/setup-cluster-env/action.yml b/.github/workflows/setup-cluster-env/action.yml
@@ -13,30 +13,34 @@ inputs:
   vault-secret-id:
     description: 'Vault secret ID'
     required: true
+  ec-key-secret:
+    default: 'secret/observability-team/ci/elastic-cloud/observability-pro'
+    description: 'EC Key secret'
+    required: false
+  aws-account-secret:
+    default: 'secret/observability-team/ci/elastic-observability-aws-account-auth'
+    description: 'AWS account secret'
+    required: false
 
 runs:
   using: "composite"
   steps:
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v4
       with:
         go-version-file: go.mod
-        cache: true
         cache-dependency-path: |
           go.sum
           tools/go.sum
-    - uses: hashicorp/[email protected]
-      env:
-        AWS_ACCOUNT_SECRET: 'secret/observability-team/ci/elastic-observability-aws-account-auth'
-        EC_KEY_SECRET: 'secret/observability-team/ci/elastic-cloud/observability-pro'
+    - uses: hashicorp/[email protected]
       with:
         url: ${{ inputs.vault-url }}
         roleId: ${{ inputs.vault-role-id }}
         secretId: ${{ inputs.vault-secret-id }}
         method: approle
         secrets: |
-          ${{ env.EC_KEY_SECRET }} apiKey | EC_API_KEY ;
-          ${{ env.AWS_ACCOUNT_SECRET }} user | AWS_PROFILE ;
-          ${{ env.AWS_ACCOUNT_SECRET }} csv | AWS_ACCOUNT_IMPORT
+          ${{ inputs.ec-key-secret }} apiKey | EC_API_KEY ;
+          ${{ inputs.aws-account-secret }} user | AWS_PROFILE ;
+          ${{ inputs.aws-account-secret }} csv | AWS_ACCOUNT_IMPORT
     - name: Configure AWS account
       run: |
         echo "${AWS_ACCOUNT_IMPORT}" > ${{ runner.temp }}/account.csv;

diff --git a/.github/workflows/smoke-tests-ess.yml b/.github/workflows/smoke-tests-ess.yml
@@ -21,7 +21,7 @@ jobs:
       tests: ${{ steps.generate.outputs.tests }}
       date: ${{ steps.generate.outputs.date }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
       - id: generate
@@ -50,7 +50,7 @@ jobs:
           - '7.17'
           - 'latest'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
       - name: Setup cluster env
@@ -66,23 +66,3 @@ jobs:
       - if: always()
         name: Teardown smoke test infra
         run: make smoketest/cleanup TEST_DIR=${{ matrix.test }}
-
-  all-smoke-tests-ess:
-    name: All Smoke Tests ESS
-    if: always()
-    runs-on: ubuntu-latest
-    needs:
-      - smoke-tests-ess
-    steps:
-      - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
-        with:
-          needs: ${{ toJSON(needs) }}
-      - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
-        with:
-          status: ${{ steps.check.outputs.status }}
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          slackChannel: "#apm-server"
-          message: "Build result for Smoke Tests ESS"
diff --git a/.github/workflows/smoke-tests-os.yml b/.github/workflows/smoke-tests-os.yml
@@ -20,7 +20,7 @@ jobs:
       tests: ${{ steps.generate.outputs.tests }}
       date: ${{ steps.generate.outputs.date }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
       - id: generate
@@ -42,7 +42,7 @@ jobs:
       TF_VAR_REPO: ${{ github.repository }}
       TF_VAR_CREATED_DATE: ${{ needs.prepare.outputs.date }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
       - name: Get version

diff --git a/.github/workflows/smoke-tests-schedule.yml b/.github/workflows/smoke-tests-schedule.yml
@@ -2,7 +2,7 @@ name: smoke-tests
 run-name: Smoke Tests
 
 on:
-  workflow_dispatch:
+  workflow_dispatch: ~
   schedule:
     - cron: '0 3 * * 1-5'
 
@@ -17,7 +17,7 @@ jobs:
     outputs:
       matrix: ${{ steps.generate.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - id: generate
         name: Generate matrix
         uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
@@ -35,12 +35,20 @@ jobs:
       branch: ${{ matrix.branch }}
     secrets: inherit
 
-  all-smoke-tests-os:
-    name: All Smoke Tests OS
+  smoke-tests-ess:
+    name: Run smoke tests ESS
+    uses: ./.github/workflows/smoke-tests-ess.yml
+    with:
+      branch: 'main'
+    secrets: inherit
+
+  all-smoke-tests:
+    name: All Smoke Tests
     if: always()
     runs-on: ubuntu-latest
     needs:
       - smoke-tests-os
+      - smoke-tests-ess
     steps:
       - id: check
         uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
@@ -53,10 +61,3 @@ jobs:
           vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
           vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
           slackChannel: "#apm-server"
-
-  smoke-tests-ess:
-    name: Run smoke tests ESS
-    uses: ./.github/workflows/smoke-tests-ess.yml
-    with:
-      branch: 'main'
-    secrets: inherit
diff --git a/.github/workflows/terraform-fmt.yml b/.github/workflows/terraform-fmt.yml
@@ -12,6 +12,6 @@ jobs:
   terraform-fmt:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: hashicorp/setup-terraform@v2
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v3
       - run: terraform fmt -recursive -check -diff
diff --git a/.github/workflows/update-beats.yml b/.github/workflows/update-beats.yml
@@ -25,8 +25,8 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.filter.outputs.matrix) }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
       - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current

diff --git a/.go-version b/.go-version
@@ -1 +1 @@
-1.20.7
+1.21.4
diff --git a/.mergify.yml b/.mergify.yml
@@ -92,6 +92,8 @@ pull_request_rules:
           - "{{ author }}"
         branches:
           - "7.17"
+        labels:
+          - "backport"
         title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})"
   - name: backport patches to 7.16 branch
     conditions:
@@ -288,3 +290,17 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})"
+  - name: backport patches to 8.11 branch
+    conditions:
+      - merged
+      - base=main
+      - label=backport-8.11
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.11"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})"
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -1,4 +1,5 @@
 include::./changelogs/head.asciidoc[]
+include::./changelogs/8.11.asciidoc[]
 include::./changelogs/8.10.asciidoc[]
 include::./changelogs/8.9.asciidoc[]
 include::./changelogs/8.8.asciidoc[]