Skip all Detection & Response Cypress tests in Serverless

x-pack/test/security_solution_cypress/cypress/README.md

@@ -40,7 +40,12 @@ of data for your test, [**Running the tests**](#running-the-tests) to know how t
 
 Please, before opening a PR with the new test, please make sure that the test fails. If you never see your test fail you don’t know if your test is actually testing the right thing, or testing anything at all.
 
-Note that we use tags in order to select which tests we want to execute: @serverless, @ess and @brokenInServerless
+Note that we use tags in order to select which tests we want to execute:
+
+- `@serverless` includes a test in the Serverless test suite. You need to explicitly add this tag to any test you want to run against a Serverless environment.
+- `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment.
+- `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken.
+- `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environemnt", "the test has been temporarily excluded, see the comment above why".
 
 Please, before opening a PR with a new test, make sure that the test fails. If you never see your test fail you don’t know if your test is actually testing the right thing, or testing anything at all.
 

x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless.config.ts

@@ -18,7 +18,7 @@ export default defineCypressConfig({
   env: {
     grepFilterSpecs: true,
     grepOmitFiltered: true,
-    grepTags: '@serverless --@brokenInServerless',
+    grepTags: '@serverless --@brokenInServerless --@skipInServerless',
   },
   execTimeout: 150000,
   pageLoadTimeout: 150000,

x-pack/test/security_solution_cypress/cypress/cypress_serverless.config.ts

@@ -23,7 +23,7 @@ export default defineCypressConfig({
   numTestsKeptInMemory: 10,
   env: {
     grepFilterSpecs: true,
-    grepTags: '@serverless --@brokenInServerless',
+    grepTags: '@serverless --@brokenInServerless --@skipInServerless',
   },
   e2e: {
     experimentalRunAllSpecs: true,

x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts

@@ -25,6 +25,7 @@ import { GET_TIMELINE_HEADER } from '../../screens/timeline';
 const alertRunTimeField = 'field.name.alert.page';
 const timelineRuntimeField = 'field.name.timeline';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe(
   'Create DataView runtime field',
   { tags: ['@ess', '@serverless', '@brokenInServerless'] },

x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts

@@ -35,12 +35,14 @@ const rolesToCreate = [secReadCasesAll];
 const siemDataViewTitle = 'Security Default Data View';
 const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*'];
 
-describe('Sourcerer', () => {
+// TODO: https://github.com/elastic/kibana/issues/161539
+describe('Sourcerer', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => {
   before(() => {
     cy.task('esArchiverResetKibana');
     dataViews.forEach((dataView: string) => postDataView(dataView));
   });
 
+  // TODO: https://github.com/elastic/kibana/issues/161539
   describe('permissions', { tags: ['@ess', '@brokenInServerless'] }, () => {
     before(() => {
       createUsersAndRoles(usersToCreate, rolesToCreate);
@@ -52,6 +54,7 @@ describe('Sourcerer', () => {
     });
   });
 
+  // TODO: https://github.com/elastic/kibana/issues/161539
   // FLAKY: https://github.com/elastic/kibana/issues/165766
   describe('Default scope', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
     beforeEach(() => {

x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer_timeline.cy.ts

@@ -38,7 +38,8 @@ import { closeTimeline, openTimelineById } from '../../tasks/timeline';
 const siemDataViewTitle = 'Security Default Data View';
 const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*'];
 
-describe('Timeline scope', { tags: '@brokenInServerless' }, () => {
+// TODO: https://github.com/elastic/kibana/issues/161539
+describe('Timeline scope', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   beforeEach(() => {
     cy.clearLocalStorage();
     login();

x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer_timeline.ts

@@ -38,7 +38,8 @@ import { closeTimeline, openTimelineById } from '../../tasks/timeline';
 const siemDataViewTitle = 'Security Default Data View';
 const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*'];
 
-describe('Timeline scope', { tags: '@brokenInServerless' }, () => {
+// TODO: https://github.com/elastic/kibana/issues/161539
+describe('Timeline scope', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   beforeEach(() => {
     cy.clearLocalStorage();
     login();

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alert_tags.cy.ts

@@ -24,6 +24,7 @@ import {
   UNSELECTED_ALERT_TAG,
 } from '../../screens/alerts';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe('Alert tagging', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   before(() => {
     cleanKibana();

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alerts_charts.cy.ts

@@ -24,6 +24,7 @@ import {
 } from '../../screens/search_bar';
 import { TOASTER } from '../../screens/alerts_detection_rules';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe(
   'Histogram legend hover actions',
   { tags: ['@ess', '@serverless', '@brokenInServerless'] },

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alerts_detection_callouts_index_outdated.cy.ts

@@ -30,9 +30,10 @@ const waitForPageTitleToBeShown = () => {
   cy.get(PAGE_TITLE).should('be.visible');
 };
 
+// TODO: https://github.com/elastic/kibana/issues/161539 Does it need to run in Serverless?
 describe(
   'Detections > Need Admin Callouts indicating an admin is needed to migrate the alert data set',
-  { tags: '@ess' },
+  { tags: ['@ess', '@skipInServerless'] },
   () => {
     before(() => {
       // First, we have to open the app on behalf of a privileged user in order to initialize it.

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/cti_enrichments.cy.ts

@@ -28,6 +28,7 @@ import { openJsonView, openThreatIndicatorDetails } from '../../tasks/alerts_det
 import { DETECTIONS_RULE_MANAGEMENT_URL } from '../../urls/navigation';
 import { addsFieldsToTimeline } from '../../tasks/rule_details';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe('CTI Enrichment', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   before(() => {
     cleanKibana();
@@ -50,6 +51,7 @@ describe('CTI Enrichment', { tags: ['@ess', '@serverless', '@brokenInServerless'
     goToRuleDetails();
   });
 
+  // TODO: https://github.com/elastic/kibana/issues/161539
   // Skipped: https://github.com/elastic/kibana/issues/162818
   it.skip('Displays enrichment matched.* fields on the timeline', () => {
     const expectedFields = {

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/enrichments.cy.ts

@@ -30,6 +30,7 @@ import { login, visit } from '../../tasks/login';
 
 import { ALERTS_URL } from '../../urls/navigation';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe('Enrichment', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   before(() => {
     cleanKibana();

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/missing_privileges_callout.cy.ts

@@ -42,7 +42,8 @@ const waitForPageTitleToBeShown = () => {
   cy.get(PAGE_TITLE).should('be.visible');
 };
 
-describe('Detections > Callouts', { tags: '@ess' }, () => {
+// TODO: https://github.com/elastic/kibana/issues/161539
+describe('Detections > Callouts', { tags: ['@ess', '@skipInServerless'] }, () => {
   before(() => {
     // First, we have to open the app on behalf of a privileged user in order to initialize it.
     // Otherwise the app will be disabled and show a "welcome"-like page.

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts

@@ -14,6 +14,7 @@ import { TIMELINE_QUERY, TIMELINE_VIEW_IN_ANALYZER } from '../../screens/timelin
 import { selectAlertsHistogram } from '../../tasks/alerts';
 import { createTimeline } from '../../tasks/timelines';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe(
   'Ransomware Detection Alerts',
   { tags: ['@ess', '@serverless', '@brokenInServerless'] },

x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts

@@ -15,6 +15,7 @@ import { selectAlertsHistogram } from '../../tasks/alerts';
 import { createTimeline } from '../../tasks/timelines';
 import { cleanKibana } from '../../tasks/common';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe(
   'Ransomware Prevention Alerts',
   { tags: ['@ess', '@serverless', '@brokenInServerless'] },

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_install_update_authorization.cy.ts

@@ -55,9 +55,11 @@ const loadPageAsReadOnlyUser = (url: string) => {
   waitForPageWithoutDateRange(url, ROLES.reader);
 };
 
+// TODO: https://github.com/elastic/kibana/issues/164451 We should find a way to make this spec work in Serverless
+// TODO: https://github.com/elastic/kibana/issues/161540
 describe(
   'Detection rules, Prebuilt Rules Installation and Update - Authorization/RBAC',
-  { tags: '@ess' },
+  { tags: ['@ess', '@serverless', '@skipInServerless'] },
   () => {
     beforeEach(() => {
       login();

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_install_update_error_handling.cy.ts

@@ -22,9 +22,10 @@ import {
   ruleUpdatesTabClick,
 } from '../../../tasks/prebuilt_rules';
 
+// TODO: https://github.com/elastic/kibana/issues/161540
 describe(
   'Detection rules, Prebuilt Rules Installation and Update - Error handling',
-  { tags: '@ess' },
+  { tags: ['@ess', '@serverless', '@skipInServerless'] },
   () => {
     beforeEach(() => {
       login();

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_install_update_workflows.cy.ts

@@ -39,9 +39,10 @@ import {
   ruleUpdatesTabClick,
 } from '../../../tasks/prebuilt_rules';
 
+// TODO: https://github.com/elastic/kibana/issues/161540
 describe(
   'Detection rules, Prebuilt Rules Installation and Update workflow',
-  { tags: ['@ess', '@brokenInServerless'] },
+  { tags: ['@ess', '@serverless', '@brokenInServerless'] },
   () => {
     beforeEach(() => {
       login();

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_management.cy.ts

@@ -50,7 +50,8 @@ const rules = Array.from(Array(5)).map((_, i) => {
   });
 });
 
-describe('Prebuilt rules', { tags: ['@ess', '@serverless'] }, () => {
+// TODO: https://github.com/elastic/kibana/issues/161540
+describe('Prebuilt rules', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => {
   before(() => {
     cleanKibana();
   });

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_notifications.cy.ts

@@ -29,9 +29,10 @@ const RULE_1 = createRuleAssetSavedObject({
   rule_id: 'rule_1',
 });
 
+// TODO: https://github.com/elastic/kibana/issues/161540
 describe(
   'Detection rules, Prebuilt Rules Installation and Update Notifications',
-  { tags: ['@ess', '@brokenInServerless'] },
+  { tags: ['@ess', '@serverless', '@brokenInServerless'] },
   () => {
     beforeEach(() => {
       login();

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_actions/rule_actions.cy.ts

@@ -27,9 +27,10 @@ import { login, visit } from '../../../tasks/login';
 
 import { RULE_CREATION } from '../../../urls/navigation';
 
+// TODO: https://github.com/elastic/kibana/issues/161539
 describe(
   'Rule actions during detection rule creation',
-  { tags: ['@ess', '@brokenInServerless'] },
+  { tags: ['@ess', '@serverless', '@brokenInServerless'] },
   () => {
     const indexConnector = getIndexConnector();
 

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/custom_query_rule.cy.ts

@@ -113,7 +113,8 @@ import { enablesRule, getDetails, waitForTheRuleToBeExecuted } from '../../../ta
 
 import { RULE_CREATION } from '../../../urls/navigation';
 
-describe('Custom query rules', { tags: ['@ess', '@brokenInServerless'] }, () => {
+// TODO: https://github.com/elastic/kibana/issues/161539
+describe('Custom query rules', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   beforeEach(() => {
     deleteAlertsAndRules();
   });

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_creation/custom_query_rule_data_view.cy.ts

@@ -67,7 +67,8 @@ import { getDetails, waitForTheRuleToBeExecuted } from '../../../tasks/rule_deta
 
 import { RULE_CREATION } from '../../../urls/navigation';
 
-describe('Custom query rules', { tags: ['@ess', '@brokenInServerless'] }, () => {
+// TODO: https://github.com/elastic/kibana/issues/161539
+describe('Custom query rules', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => {
   describe('Custom detection rules creation with data views', () => {
     const rule = getDataViewRule();
     const expectedUrls = rule.references?.join('');