[Security Solution] Specific Cypress executions for `Detection Engine…

…` team (elastic#172298) Co-authored-by: Charlie Pichette <[email protected]>
ashokaditya · Dec 4, 2023 · 217a663 · 217a663
1 parent 7d3cbd2
commit 217a663
Show file tree

Hide file tree

Showing 66 changed files with 760 additions and 1,425 deletions.
diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 6
+        parallelism: 2
         retry:
           automatic:
             - exit_status: '*'
@@ -115,6 +115,32 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
+        parallelism: 4
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+        label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+        label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
         parallelism: 6
         retry:
           automatic:

diff --git a/.buildkite/pipelines/flaky_tests/groups.json b/.buildkite/pipelines/flaky_tests/groups.json
@@ -41,6 +41,23 @@
       "key": "cypress/security_serverless_rule_management_prebuilt_rules",
       "name": "[Serverless] Security Solution Rule Management - Prebuilt Rules - Cypress"
     },
+    {
+      "key": "cypress/security_solution_detection_engine",
+      "name": "Security Solution Detection Engine - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_detection_engine",
+      "name": "[Serverless] Security Solution Detection Engine - Cypress"
+    },
+
+    {
+      "key": "cypress/security_solution_detection_engine_exceptions",
+      "name": "Security Solution Detection Engine - Exceptions - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_detection_engine_exceptions",
+      "name": "[Serverless] Security Solution Detection Engine - Exceptions - Cypress"
+    },
     {
       "key": "cypress/defend_workflows",
       "name": "Security Solution Defend Workflows - Cypress"

diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -85,7 +85,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
@@ -133,7 +133,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'
@@ -163,13 +163,61 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+    label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+    label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
+    label: 'Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
+    label: 'Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 16
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '*'

diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -63,7 +63,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
@@ -111,6 +111,30 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+    label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+    label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
     parallelism: 6
     retry:
       automatic:
@@ -123,7 +147,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 16
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '*'
@@ -164,6 +188,30 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
+    label: 'Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
+    label: 'Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'

diff --git a/.buildkite/pipelines/security_solution/security_solution_cypress.yml b/.buildkite/pipelines/security_solution/security_solution_cypress.yml
@@ -54,6 +54,30 @@ steps:
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine
+    label: 'Serverless MKI QA Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions
+    label: 'Serverless MKI QA Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'

diff --git a/.buildkite/scripts/steps/functional/security_serverless_detection_engine.sh b/.buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Detection Engine Cypress Tests on Serverless"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:detection_engine:run:serverless; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh b/.buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Detection Engine - Exceptions - Cypress Tests on Serverless"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:detection_engine:exceptions:run:serverless; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/security_solution_detection_engine.sh b/.buildkite/scripts/steps/functional/security_solution_detection_engine.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Detection Engine - Security Solution Cypress Tests"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:detection_engine:run:ess; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh b/.buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Detection Engine - Exceptions - Security Solution Cypress Tests"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:detection_engine:exceptions:run:ess; status=$?; yarn junit:merge || :; exit $status
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1386,14 +1386,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals @elastic/security-detection-engine
 /x-pack/plugins/security_solution/server/lib/sourcerer @elastic/security-detection-engine
 
-/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/sourcerer @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_actions @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_edit @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/value_lists @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/exceptions @elastic/security-detection-engine
-/x-pack/test/security_solution_cypress/cypress/e2e/overview @elastic/security-detection-engine
+/x-pack/test/security_solution_cypress/cypress/e2e/detection_engine @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions  @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions @elastic/security-detection-engine

diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md
@@ -62,15 +62,20 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio
 | cypress | Runs the default Cypress command |
 | cypress:open:ess | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a local kibana and ES instance. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. |
 | cypress:open:serverless | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a mocked serverless environment. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. |
-| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations`,`explore` and `detection_response/rule_management` directories in headless mode |
+| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations`,`explore` and `detection_response` directories in headless mode |
 | cypress:run:cases:ess | Runs all tests under `explore/cases` in the `e2e` directory related to the Cases area team in headless mode |
 | cypress:ess | Runs all ESS tests with the specified configuration in headless mode and produces a report using `cypress-multi-reporters` |
+
 | cypress:rule_management:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode |
 | cypress:rule_management:prebuilt_rules:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode |
 | cypress:run:respops:ess | Runs all tests related to the Response Ops area team, specifically tests in `detection_alerts`, `detection_rules`, and `exceptions` directories in headless mode |
-| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode |
+| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations`, `explore` and `detections_response` directories in headless mode |
 | cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode |
 | cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode |
+| cypress:detection_engine:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/detection_engine` excluding `e2e/detection_response/detection_engine/exceptions` directory in headless mode |
+| cypress:detection_engine:exceptions:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/detection_engine/exceptions` directory in headless mode |
+| cypress:detection_engine:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/detection_engine` excluding `e2e/detection_response/detection_engine` directory in headless mode |
+| cypress:detection_engine:exceptions:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/detection_engine/exceptions` directory in headless mode |
 | cypress:investigations:run:ess | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode |
 | cypress:explore:run:ess | Runs all tests tagged as ESS in the `e2e/explore` directory in headless mode |
 | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode |
@@ -81,6 +86,10 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio
 | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. |
 | cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. |
 | cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. |
+| cypress:run:qa:serverless:detection_engine | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/detection_engine` directory, excluding `e2e/detection_response/detection_engine/exceptions` in headless mode using the QA environment and reak MKI projects. |
+| cypress:run:qa:serverless:detection_engine:exceptions | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/detection_engine/exceptions` directory in headless mode using the QA environment and reak MKI projects. |
+
+
 | junit:merge | Merges individual test reports into a single report and moves the report to the `junit` directory |
 
 Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging.
@@ -111,6 +120,7 @@ If you belong to one of the teams listed in the table, please add new e2e specs
 | `e2e/explore` | Threat Hunting Explore |
 | `e2e/investigations` | Threat Hunting Investigations |
 | `e2e/detection_response/rule_management` | Detection Rule Management |
+| `e2e/detection_response/detection_engine` | Detection Engine |
 
 ### fixtures/
 
@@ -259,6 +269,9 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio
 | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. |
 | cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. |
 | cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. |
+| cypress:run:qa:serverless:detection_engine | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/detection_engine` directory, excluding `e2e/detection_response/detection_engine/exceptions` in headless mode using the QA environment and reak MKI projects. |
+| cypress:run:qa:serverless:detection_engine:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/detection_engine/exceptions` directory in headless mode using the QA environment and reak MKI projects. |
+
 
 Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging.