[ci] Move cypress burn tests out of base pull request pipeline (elast…

…ic#169326)

Currently operations does not have a good mechanism for communicating
soft failures, leading to confusion on pull requests. We're tracking
support for soft failures at
elastic#169244.

For example, in elastic#169309, it
doesn't makes sense for the author to be responsible for fixing
flakiness here, so we need to make it clear it can be ignored or find a
way to skip this step in similar situations.

This moves the cypress burn out of the base pull request pipeline,
behind the `ci:all-cypress-suites` and `ci:cypress-burn` labels. I'm
open to all improvements here, not sure this quite finds the right
balance between stability and noise.

.buildkite/pipelines/pull_request/base.yml

@@ -141,17 +141,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/defend_workflows_burn.sh
-    label: 'Defend Workflows Cypress Tests, burning changed specs'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 60
-    soft_fail: true
-    parallelism: 1
-    retry:
-      automatic: false
-
   - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
     label: 'Defend Workflows Cypress Tests on Serverless'
     agents:
@@ -164,17 +153,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh
-    label: 'Defend Workflows Cypress Tests on Serverless, burning changed specs'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 60
-    soft_fail: true
-    parallelism: 1
-    retry:
-      automatic: false
-
   - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
     label: 'Threat Intelligence Cypress Tests'
     agents:
@@ -199,27 +177,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
-    label: 'Security Solution Cypress tests, burning changed specs'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 1
-    retry:
-      automatic: false
-    soft_fail: true
-
-  - command: .buildkite/scripts/steps/functional/osquery_cypress_burn.sh
-    label: 'Osquery Cypress Tests, burning changed specs'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 50
-    soft_fail: true
-    retry:
-      automatic: false
-
   - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
     label: 'Serverless Osquery Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/cypress_burn.yml

@@ -0,0 +1,43 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/defend_workflows_burn.sh
+    label: 'Defend Workflows Cypress Tests, burning changed specs'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    soft_fail: true
+    parallelism: 1
+    retry:
+      automatic: false
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh
+    label: 'Defend Workflows Cypress Tests on Serverless, burning changed specs'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    soft_fail: true
+    parallelism: 1
+    retry:
+      automatic: false
+
+  - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
+    label: 'Security Solution Cypress tests, burning changed specs'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic: false
+    soft_fail: true
+
+  - command: .buildkite/scripts/steps/functional/osquery_cypress_burn.sh
+    label: 'Osquery Cypress Tests, burning changed specs'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 50
+    soft_fail: true
+    retry:
+      automatic: false

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -164,6 +164,13 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/check_next_docs.yml'));
     }
 
+    if (
+      GITHUB_PR_LABELS.includes('ci:cypress-burn') ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/cypress_burn.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     // remove duplicated steps