Update heartbeat pipeline to match Jenkins (elastic#39334)

Updated heartbeat pipeline to match same steps & commands as in Jenkins

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -46,7 +46,7 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Ubuntu Unit Tests"
+              context: "heartbeat: Ubuntu Unit Tests"
 
       - label: ":rhel: Heartbeat Rhel9 Unit Tests"
         command: |
@@ -64,9 +64,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Rhel9 Unit Tests"
+              context: "heartbeat: Rhel9 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-2016 Unit Tests"
+      - label: ":windows: Win-2016 Unit Tests"
         key: "windows-2016"
         command: |
           Set-Location -Path heartbeat
@@ -84,7 +84,7 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2016 Unit Tests"
+              context: "heartbeat: Win-2016 Unit Tests"
 
       - label: ":windows: Heartbeat Win-2022 Unit Test"
         key: "windows-2022"
@@ -104,10 +104,19 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2022 Unit Tests"
+              context: "heartbeat: Win-2022 Unit Tests"
 
-      - label: ":ubuntu: Heartbeat Go Integration Tests"
+      - label: ":ubuntu: Heartbeat Go (Module) Integration Tests"
         command: |
+          set -euo pipefail
+          echo "~~~ Installing @elastic/synthetics"
+          npm install -g @elastic/synthetics
+
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet heartbeat
+
+          echo "~~~ Running tests"
           cd heartbeat
           mage goIntegTest
         retry:
@@ -122,10 +131,16 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Go Integration Tests"
+              context: "heartbeat: Go Integration Tests"
 
-      - label: ":ubuntu: Heartbeat Python Integration Tests"
+      - label: ":ubuntu: Heartbeat Python (Module) Integration Tests"
         command: |
+          set -euo pipefail
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet heartbeat
+
+          echo "~~~ Running tests"
           cd heartbeat
           mage pythonIntegTest
         retry:
@@ -140,15 +155,16 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Python Integration Tests"
+              context: "heartbeat: Python Integration Tests"
 
-  - group: "Heartbeat ARM Tests"
-    key: "heartbeat-extended-tests-arm"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+  - group: "Heartbeat Extended Tests"
+    key: "heartbeat-extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
 
     steps:
-      - label: ":linux: Heartbeat ARM64 Unit Tests"
+      - label: ":linux: Heartbeat arm64 Unit Tests"
         key: "arm-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
           cd heartbeat
           mage build unitTest
@@ -162,14 +178,11 @@ steps:
         artifact_paths: "heartbeat/build/*.xml"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Ubuntu ARM64 Unit Tests"
+              context: "heartbeat: Linux arm64 Unit Tests"
 
-  - group: "Heartbeat Extended Testing MacOS"
-    key: "heartbeat-extended-tests-macos"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-    steps:
-      - label: ":mac: Heartbeat MacOS Unit Tests"
+      - label: ":mac: macOS x86_64 Unit Tests"
         key: "macos-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -186,10 +199,11 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: MacOS Unit Tests"
+              context: "heartbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: Heartbeat MacOS ARM Unit Tests"
+      - label: ":mac: macOS arm64 Unit Tests"
         key: "macos-arm-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -206,14 +220,14 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: MacOS ARM Unit Tests"
+              context: "heartbeat: macOS arm64 Unit Tests"
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
 
     steps:
-      - label: ":windows: Heartbeat Win-2019 Unit Tests"
+      - label: ":windows: Win-2019 Unit Tests"
         key: "heartbeat-win-extended-2019"
         command: |
           Set-Location -Path heartbeat
@@ -231,9 +245,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-2019 Unit Tests"
+              context: "heartbeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-11 Unit Tests"
+      - label: ":windows: Win-11 Unit Tests"
         key: "heartbeat-windows-extended-11"
         command: |
           Set-Location -Path heartbeat
@@ -251,9 +265,9 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-11 Unit Tests"
+              context: "heartbeat: Win-11 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-10 Unit Tests"
+      - label: ":windows: Win-10 Unit Tests"
         key: "heartbeat-windows-extended-10"
         command: |
           Set-Location -Path heartbeat
@@ -271,17 +285,20 @@ steps:
           - "heartbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Win-10 Unit Tests"
+              context: "heartbeat: Win-10 Unit Tests"
 
   - wait: ~
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on: "heartbeat-mandatory-tests"
 
   - group: "Heartbeat Packaging"
     key: "heartbeat-packaging"
     steps:
-      - label: ":ubuntu: Heartbeat Packaging Linux X86"
-        key: "heartbeat-package-linux-x86"
+      - label: ":ubuntu: Heartbeat Packaging Linux"
+        key: "heartbeat-package-linux"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
@@ -298,9 +315,9 @@ steps:
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Packaging Ubuntu x86_64"
+              context: "heartbeat: Packaging Linux"
 
-      - label: ":linux: Heartbeat Packaging Linux ARM"
+      - label: ":linux: Heartbeat Packaging Linux arm64"
         key: "heartbeat-package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
@@ -319,4 +336,4 @@ steps:
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         notify:
           - github_commit_status:
-              context: "Heartbeat: Packaging Ubuntu ARM"
+              context: "heartbeat: Packaging Linux arm64"