From b06b1e3c80ad8a026f21466b2397b259a90fb88c Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 3 Dec 2024 13:01:09 +0100
Subject: [PATCH 01/13] [kbn/scout] update script to use tagging

---
 packages/kbn-scout/index.ts                   |   2 +-
 .../kbn-scout/src/playwright/constants.ts     |  27 +++
 .../src/playwright/fixtures/test/index.ts     |   4 +-
 .../playwright/fixtures/test/validate_tags.ts |  35 ++++
 packages/kbn-scout/src/playwright/index.ts    |   3 +
 .../src/playwright/runner/run_tests.ts        |  12 +-
 .../ui_tests/tests/error_handling.spec.ts     |   4 +-
 .../tests/saved_search_embeddable.spec.ts     | 108 ++++++-----
 .../ui_tests/tests/saved_searches.spec.ts     | 173 +++++++++---------
 .../ui_tests/tests/value_suggestions.spec.ts  |   4 +-
 .../value_suggestions_non_time_based.spec.ts  |   4 +-
 ...uggestions_use_time_range_disabled.spec.ts |   4 +-
 12 files changed, 223 insertions(+), 157 deletions(-)
 create mode 100644 packages/kbn-scout/src/playwright/constants.ts
 create mode 100644 packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts

diff --git a/packages/kbn-scout/index.ts b/packages/kbn-scout/index.ts
index 34a906fcf755d..d6489694fd06a 100644
--- a/packages/kbn-scout/index.ts
+++ b/packages/kbn-scout/index.ts
@@ -8,7 +8,7 @@
  */
 
 export { startServersCli, runTestsCli } from './src/cli';
-export { expect, test, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
+export { expect, test, tags, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
 export type {
   ScoutPage,
   ScoutPlaywrightOptions,
diff --git a/packages/kbn-scout/src/playwright/constants.ts b/packages/kbn-scout/src/playwright/constants.ts
new file mode 100644
index 0000000000000..a022b6afd3430
--- /dev/null
+++ b/packages/kbn-scout/src/playwright/constants.ts
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+const SERVERLESS_ONLY = ['@svlSecurity', '@svlOblt', '@svlSearch'];
+const ESS_ONLY = ['@ess'];
+const DEPLOYMENT_AGNOSTIC = SERVERLESS_ONLY.concat(ESS_ONLY);
+
+export const tags = {
+  ESS_ONLY,
+  SERVERLESS_ONLY,
+  DEPLOYMENT_AGNOSTIC,
+};
+
+export const tagsByMode = {
+  stateful: '@ess',
+  serverless: {
+    es: '@svlSearch',
+    oblt: '@svlOblt',
+    security: '@svlSecurity',
+  },
+};
diff --git a/packages/kbn-scout/src/playwright/fixtures/test/index.ts b/packages/kbn-scout/src/playwright/fixtures/test/index.ts
index 41bfedcf39dc7..3e2ea85b90bc3 100644
--- a/packages/kbn-scout/src/playwright/fixtures/test/index.ts
+++ b/packages/kbn-scout/src/playwright/fixtures/test/index.ts
@@ -11,9 +11,11 @@ import { mergeTests } from '@playwright/test';
 import { browserAuthFixture } from './browser_auth';
 import { scoutPageFixture } from './page';
 import { pageObjectsFixture } from './page_objects';
+import { validateTagsFixture } from './validate_tags';
 
 export const scoutTestFixtures = mergeTests(
   browserAuthFixture,
   scoutPageFixture,
-  pageObjectsFixture
+  pageObjectsFixture,
+  validateTagsFixture
 );
diff --git a/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts b/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
new file mode 100644
index 0000000000000..6db0b2d207cde
--- /dev/null
+++ b/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { test as base } from '@playwright/test';
+import { tags } from '../../constants';
+
+const supportedTags = tags.DEPLOYMENT_AGNOSTIC;
+
+export const validateTagsFixture = base.extend<{ validateTags: void }>({
+  validateTags: [
+    async ({}, use, testInfo) => {
+      if (testInfo.tags.length === 0) {
+        throw new Error(`At least one tag is required: ${supportedTags.join(', ')}`);
+      }
+
+      const invalidTags = testInfo.tags.filter((tag: string) => !supportedTags.includes(tag));
+      if (invalidTags.length > 0) {
+        throw new Error(
+          `Unsupported tag(s) found in test suite "${testInfo.title}": ${invalidTags.join(
+            ', '
+          )}. ` + `Supported tags are: ${supportedTags.join(', ')}.`
+        );
+      }
+
+      await use();
+    },
+    { auto: true },
+  ],
+});
diff --git a/packages/kbn-scout/src/playwright/index.ts b/packages/kbn-scout/src/playwright/index.ts
index 5294274c41bc5..c12dbd06444c4 100644
--- a/packages/kbn-scout/src/playwright/index.ts
+++ b/packages/kbn-scout/src/playwright/index.ts
@@ -25,3 +25,6 @@ export type {
   ScoutPage,
   EsArchiverFixture,
 } from './fixtures';
+
+// use to tag tests
+export { tags } from './constants';
diff --git a/packages/kbn-scout/src/playwright/runner/run_tests.ts b/packages/kbn-scout/src/playwright/runner/run_tests.ts
index a5d8aa137dbfd..9463e6fa201e5 100644
--- a/packages/kbn-scout/src/playwright/runner/run_tests.ts
+++ b/packages/kbn-scout/src/playwright/runner/run_tests.ts
@@ -18,12 +18,17 @@ import { loadServersConfig } from '../../config';
 import { silence } from '../../common';
 import { RunTestsOptions } from './flags';
 import { getExtraKbnOpts } from '../../servers/run_kibana_server';
+import { tagsByMode } from '../constants';
 
 export async function runTests(log: ToolingLog, options: RunTestsOptions) {
   const runStartTime = Date.now();
   const reportTime = getTimeReporter(log, 'scripts/scout_test');
 
   const config = await loadServersConfig(options.mode, log);
+
+  const playwrightTag = config.get('serverless')
+    ? tagsByMode.serverless[config.get('projectType') as keyof typeof tagsByMode.serverless]
+    : tagsByMode.stateful;
   const playwrightConfigPath = options.configPath;
 
   await withProcRunner(log, async (procs) => {
@@ -59,7 +64,12 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
       // Running 'npx playwright test --config=${playwrightConfigPath}'
       await procs.run(`playwright`, {
         cmd: resolve(REPO_ROOT, './node_modules/.bin/playwright'),
-        args: ['test', `--config=${playwrightConfigPath}`, ...(options.headed ? ['--headed'] : [])],
+        args: [
+          'test',
+          `--config=${playwrightConfigPath}`,
+          `--grep=${playwrightTag}`,
+          ...(options.headed ? ['--headed'] : []),
+        ],
         cwd: resolve(REPO_ROOT),
         env: {
           ...process.env,
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/error_handling.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/error_handling.spec.ts
index 914558fbdc97f..82c45b52ad567 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/error_handling.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/error_handling.spec.ts
@@ -5,10 +5,10 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 
-test.describe('Discover app - errors', { tag: ['@ess'] }, () => {
+test.describe('Discover app - errors', { tag: tags.ESS_ONLY }, () => {
   test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
     await kbnClient.savedObjects.clean({ types: ['search', 'index-pattern'] });
     await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_search_embeddable.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_search_embeddable.spec.ts
index 7103f2b25e633..6c37611dbc202 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_search_embeddable.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_search_embeddable.spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ScoutWorkerFixtures, expect } from '@kbn/scout';
+import { ScoutWorkerFixtures, expect, tags } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 
 const createSavedSearch = async (
@@ -37,66 +37,62 @@ const createSavedSearch = async (
     ],
   });
 
-test.describe(
-  'Discover app - saved search embeddable',
-  { tag: ['@ess', '@svlSecurity', '@svlOblt', '@svlSearch'] },
-  () => {
-    const SAVED_SEARCH_TITLE = 'TempSearch';
-    const SAVED_SEARCH_ID = '90943e30-9a47-11e8-b64d-95841ca0b247';
-    test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
-      await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);
-      await kbnClient.importExport.load(testData.KBN_ARCHIVES.DASHBOARD_DRILLDOWNS);
-      await uiSettings.set({
-        defaultIndex: testData.DATA_VIEW_ID.LOGSTASH, // TODO: investigate why it is required for `node scripts/playwright_test.js` run
-        'timepicker:timeDefaults': `{ "from": "${testData.LOGSTASH_DEFAULT_START_TIME}", "to": "${testData.LOGSTASH_DEFAULT_END_TIME}"}`,
-      });
+test.describe('Discover app - saved search embeddable', { tag: tags.DEPLOYMENT_AGNOSTIC }, () => {
+  const SAVED_SEARCH_TITLE = 'TempSearch';
+  const SAVED_SEARCH_ID = '90943e30-9a47-11e8-b64d-95841ca0b247';
+  test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
+    await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);
+    await kbnClient.importExport.load(testData.KBN_ARCHIVES.DASHBOARD_DRILLDOWNS);
+    await uiSettings.set({
+      defaultIndex: testData.DATA_VIEW_ID.LOGSTASH, // TODO: investigate why it is required for `node scripts/playwright_test.js` run
+      'timepicker:timeDefaults': `{ "from": "${testData.LOGSTASH_DEFAULT_START_TIME}", "to": "${testData.LOGSTASH_DEFAULT_END_TIME}"}`,
     });
+  });
 
-    test.afterAll(async ({ kbnClient, uiSettings }) => {
-      await uiSettings.unset('defaultIndex', 'timepicker:timeDefaults');
-      await kbnClient.savedObjects.cleanStandardList();
-    });
+  test.afterAll(async ({ kbnClient, uiSettings }) => {
+    await uiSettings.unset('defaultIndex', 'timepicker:timeDefaults');
+    await kbnClient.savedObjects.cleanStandardList();
+  });
 
-    test.beforeEach(async ({ browserAuth, pageObjects }) => {
-      await browserAuth.loginAsPrivilegedUser();
-      await pageObjects.dashboard.goto();
-    });
+  test.beforeEach(async ({ browserAuth, pageObjects }) => {
+    await browserAuth.loginAsPrivilegedUser();
+    await pageObjects.dashboard.goto();
+  });
 
-    test('should allow removing the dashboard panel after the underlying saved search has been deleted', async ({
+  test('should allow removing the dashboard panel after the underlying saved search has been deleted', async ({
+    kbnClient,
+    page,
+    pageObjects,
+  }) => {
+    await pageObjects.dashboard.openNewDashboard();
+    await createSavedSearch(
       kbnClient,
-      page,
-      pageObjects,
-    }) => {
-      await pageObjects.dashboard.openNewDashboard();
-      await createSavedSearch(
-        kbnClient,
-        SAVED_SEARCH_ID,
-        SAVED_SEARCH_TITLE,
-        testData.DATA_VIEW_ID.LOGSTASH
-      );
-      await pageObjects.dashboard.addPanelFromLibrary(SAVED_SEARCH_TITLE);
-      await page.testSubj.locator('savedSearchTotalDocuments').waitFor({
-        state: 'visible',
-      });
+      SAVED_SEARCH_ID,
+      SAVED_SEARCH_TITLE,
+      testData.DATA_VIEW_ID.LOGSTASH
+    );
+    await pageObjects.dashboard.addPanelFromLibrary(SAVED_SEARCH_TITLE);
+    await page.testSubj.locator('savedSearchTotalDocuments').waitFor({
+      state: 'visible',
+    });
 
-      await pageObjects.dashboard.saveDashboard('Dashboard with deleted saved search');
-      await kbnClient.savedObjects.delete({
-        type: 'search',
-        id: SAVED_SEARCH_ID,
-      });
+    await pageObjects.dashboard.saveDashboard('Dashboard with deleted saved search');
+    await kbnClient.savedObjects.delete({
+      type: 'search',
+      id: SAVED_SEARCH_ID,
+    });
 
-      await page.reload();
-      await page.waitForLoadingIndicatorHidden();
-      await expect(
-        page.testSubj.locator('embeddableError'),
-        'Embeddable error should be displayed'
-      ).toBeVisible();
+    await page.reload();
+    await page.waitForLoadingIndicatorHidden();
+    await expect(
+      page.testSubj.locator('embeddableError'),
+      'Embeddable error should be displayed'
+    ).toBeVisible();
 
-      await pageObjects.dashboard.removePanel('embeddableError');
-      await expect(
-        page.testSubj.locator('embeddableError'),
-        'Embeddable error should not be displayed'
-      ).toBeHidden();
-    });
-  }
-);
+    await pageObjects.dashboard.removePanel('embeddableError');
+    await expect(
+      page.testSubj.locator('embeddableError'),
+      'Embeddable error should not be displayed'
+    ).toBeHidden();
+  });
+});
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts
index 184c9217bfe15..5b18bc460e935 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 import type { ExtendedScoutTestFixtures } from '../fixtures';
 
@@ -31,99 +31,92 @@ const assertDataViewIsSelected = async (page: ExtendedScoutTestFixtures['page'],
     'Incorrect data view is selected'
   ).toHaveText(name);
 
-test.describe(
-  'Discover app - saved searches',
-  { tag: ['@ess', '@svlSecurity', '@svlOblt', '@svlSearch'] },
-  () => {
-    const START_TIME = '2019-04-27T23:56:51.374Z';
-    const END_TIME = '2019-08-23T16:18:51.821Z';
-    const PANEL_NAME = 'Ecommerce Data';
-    const SEARCH_QUERY = 'customer_gender:MALE';
-    const SAVED_SEARCH_NAME = 'test-unselect-saved-search';
-    const filterFieldAndValue = {
-      field: 'category',
-      value: `Men's Shoes`,
-    };
-
-    test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
-      await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.ECOMMERCE);
-      await kbnClient.importExport.load(testData.KBN_ARCHIVES.DISCOVER);
-      await kbnClient.importExport.load(testData.KBN_ARCHIVES.ECOMMERCE);
-      await uiSettings.set({
-        defaultIndex: testData.DATA_VIEW_ID.ECOMMERCE,
-        'doc_table:legacy': false,
-        'timepicker:timeDefaults': `{ "from": "${START_TIME}", "to": "${END_TIME}"}`,
-      });
+test.describe('Discover app - saved searches', { tag: tags.DEPLOYMENT_AGNOSTIC }, () => {
+  const START_TIME = '2019-04-27T23:56:51.374Z';
+  const END_TIME = '2019-08-23T16:18:51.821Z';
+  const PANEL_NAME = 'Ecommerce Data';
+  const SEARCH_QUERY = 'customer_gender:MALE';
+  const SAVED_SEARCH_NAME = 'test-unselect-saved-search';
+  const filterFieldAndValue = {
+    field: 'category',
+    value: `Men's Shoes`,
+  };
+
+  test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
+    await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.ECOMMERCE);
+    await kbnClient.importExport.load(testData.KBN_ARCHIVES.DISCOVER);
+    await kbnClient.importExport.load(testData.KBN_ARCHIVES.ECOMMERCE);
+    await uiSettings.set({
+      defaultIndex: testData.DATA_VIEW_ID.ECOMMERCE,
+      'doc_table:legacy': false,
+      'timepicker:timeDefaults': `{ "from": "${START_TIME}", "to": "${END_TIME}"}`,
     });
-
-    test.afterAll(async ({ kbnClient, uiSettings }) => {
-      await uiSettings.unset('doc_table:legacy', 'defaultIndex', 'timepicker:timeDefaults');
-      await kbnClient.savedObjects.cleanStandardList();
+  });
+
+  test.afterAll(async ({ kbnClient, uiSettings }) => {
+    await uiSettings.unset('doc_table:legacy', 'defaultIndex', 'timepicker:timeDefaults');
+    await kbnClient.savedObjects.cleanStandardList();
+  });
+
+  test.beforeEach(async ({ browserAuth }) => {
+    await browserAuth.loginAsPrivilegedUser();
+  });
+
+  test('should customize time range on dashboards', async ({ pageObjects, page }) => {
+    await pageObjects.dashboard.goto();
+    await pageObjects.dashboard.openNewDashboard();
+    await pageObjects.dashboard.addPanelFromLibrary(PANEL_NAME);
+    await page.testSubj.locator('savedSearchTotalDocuments').waitFor({
+      state: 'visible',
     });
 
-    test.beforeEach(async ({ browserAuth }) => {
-      await browserAuth.loginAsPrivilegedUser();
+    await pageObjects.dashboard.customizePanel({
+      name: PANEL_NAME,
+      customTimeRageCommonlyUsed: { value: 'Last_90 days' },
     });
-
-    test('should customize time range on dashboards', async ({ pageObjects, page }) => {
-      await pageObjects.dashboard.goto();
-      await pageObjects.dashboard.openNewDashboard();
-      await pageObjects.dashboard.addPanelFromLibrary(PANEL_NAME);
-      await page.testSubj.locator('savedSearchTotalDocuments').waitFor({
-        state: 'visible',
-      });
-
-      await pageObjects.dashboard.customizePanel({
-        name: PANEL_NAME,
-        customTimeRageCommonlyUsed: { value: 'Last_90 days' },
-      });
-      await expect(
-        page.testSubj.locator('embeddedSavedSearchDocTable').locator('.euiDataGrid__noResults'),
-        'No results message in Saved Search panel should be visible'
-      ).toBeVisible();
+    await expect(
+      page.testSubj.locator('embeddedSavedSearchDocTable').locator('.euiDataGrid__noResults'),
+      'No results message in Saved Search panel should be visible'
+    ).toBeVisible();
+  });
+
+  test(`should unselect saved search when navigating to a 'new'`, async ({ pageObjects, page }) => {
+    await pageObjects.discover.goto();
+    await assertDataViewIsSelected(page, testData.DATA_VIEW.ECOMMERCE);
+    await pageObjects.filterBar.addFilter({
+      ...filterFieldAndValue,
+      operator: 'is',
     });
+    await page.testSubj.fill('queryInput', SEARCH_QUERY);
+    await page.testSubj.click('querySubmitButton');
+    await pageObjects.discover.waitForHistogramRendered();
 
-    test(`should unselect saved search when navigating to a 'new'`, async ({
-      pageObjects,
-      page,
-    }) => {
-      await pageObjects.discover.goto();
-      await assertDataViewIsSelected(page, testData.DATA_VIEW.ECOMMERCE);
-      await pageObjects.filterBar.addFilter({
+    await pageObjects.discover.saveSearch(SAVED_SEARCH_NAME);
+    await pageObjects.discover.waitForHistogramRendered();
+
+    expect(
+      await pageObjects.filterBar.hasFilter({
         ...filterFieldAndValue,
-        operator: 'is',
-      });
-      await page.testSubj.fill('queryInput', SEARCH_QUERY);
-      await page.testSubj.click('querySubmitButton');
-      await pageObjects.discover.waitForHistogramRendered();
-
-      await pageObjects.discover.saveSearch(SAVED_SEARCH_NAME);
-      await pageObjects.discover.waitForHistogramRendered();
-
-      expect(
-        await pageObjects.filterBar.hasFilter({
-          ...filterFieldAndValue,
-          enabled: true, // Filter is enabled by default
-        })
-      ).toBe(true);
-      await expect(page.testSubj.locator('queryInput')).toHaveText(SEARCH_QUERY);
-
-      // create new search
-      await pageObjects.discover.clickNewSearch();
-      await assertDataViewIsSelected(page, testData.DATA_VIEW.ECOMMERCE);
-      await assertNoFilterAndEmptyQuery(filterFieldAndValue, pageObjects, page);
-
-      // change data view
-      await pageObjects.discover.selectDataView(testData.DATA_VIEW.LOGSTASH);
-      await assertNoFilterAndEmptyQuery(filterFieldAndValue, pageObjects, page);
-
-      // change data view again
-      await pageObjects.discover.selectDataView(testData.DATA_VIEW.ECOMMERCE);
-      await assertNoFilterAndEmptyQuery(filterFieldAndValue, pageObjects, page);
-
-      // create new search again
-      await pageObjects.discover.clickNewSearch();
-      await assertDataViewIsSelected(page, testData.DATA_VIEW.ECOMMERCE);
-    });
-  }
-);
+        enabled: true, // Filter is enabled by default
+      })
+    ).toBe(true);
+    await expect(page.testSubj.locator('queryInput')).toHaveText(SEARCH_QUERY);
+
+    // create new search
+    await pageObjects.discover.clickNewSearch();
+    await assertDataViewIsSelected(page, testData.DATA_VIEW.ECOMMERCE);
+    await assertNoFilterAndEmptyQuery(filterFieldAndValue, pageObjects, page);
+
+    // change data view
+    await pageObjects.discover.selectDataView(testData.DATA_VIEW.LOGSTASH);
+    await assertNoFilterAndEmptyQuery(filterFieldAndValue, pageObjects, page);
+
+    // change data view again
+    await pageObjects.discover.selectDataView(testData.DATA_VIEW.ECOMMERCE);
+    await assertNoFilterAndEmptyQuery(filterFieldAndValue, pageObjects, page);
+
+    // create new search again
+    await pageObjects.discover.clickNewSearch();
+    await assertDataViewIsSelected(page, testData.DATA_VIEW.ECOMMERCE);
+  });
+});
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts
index f80daeea5b9c7..65c4ae2144e8e 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts
@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData, assertionMessages } from '../fixtures';
 
 test.describe(
   'Discover app - value suggestions: useTimeRange enabled',
-  { tag: ['@ess', '@svlSecurity', '@svlOblt', '@svlSearch'] },
+  { tag: tags.DEPLOYMENT_AGNOSTIC },
   () => {
     test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
       await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts
index 0e4591ffd9e1e..4c2c452ac0ae7 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts
@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData, assertionMessages } from '../fixtures';
 
 test.describe(
   'Discover app - value suggestions non-time based',
-  { tag: ['@ess', '@svlSecurity', '@svlOblt', '@svlSearch'] },
+  { tag: tags.DEPLOYMENT_AGNOSTIC },
   () => {
     test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
       await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.NO_TIME_FIELD);
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_use_time_range_disabled.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_use_time_range_disabled.spec.ts
index c5f0ad9b43ef6..c9ea665cd3686 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_use_time_range_disabled.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_use_time_range_disabled.spec.ts
@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData, assertionMessages } from '../fixtures';
 
 test.describe(
   'Discover app - value suggestions: useTimeRange disabled',
-  { tag: ['@ess', '@svlSecurity', '@svlOblt', '@svlSearch'] },
+  { tag: tags.DEPLOYMENT_AGNOSTIC },
   () => {
     test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
       await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);

From 2196b50507cd8ad1d49718c27735e2e0bf495b3c Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 3 Dec 2024 13:41:34 +0100
Subject: [PATCH 02/13] [ci] add pipeline to run stateful tests

---
 .../pipelines/pull_request/scout_ui_tests.yml | 19 +++++++++++++++++++
 .../pipelines/pull_request/pipeline.ts        | 10 ++++++++++
 .../steps/functional/scout_ui_tests.sh        | 12 ++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 .buildkite/pipelines/pull_request/scout_ui_tests.yml
 create mode 100755 .buildkite/scripts/steps/functional/scout_ui_tests.sh

diff --git a/.buildkite/pipelines/pull_request/scout_ui_tests.yml b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
new file mode 100644
index 0000000000000..cdbe3b0b02e42
--- /dev/null
+++ b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
@@ -0,0 +1,19 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
+    label: 'Scout UI Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+      - check_oas_snapshot
+    timeout_in_minutes: 20
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.ts b/.buildkite/scripts/pipelines/pull_request/pipeline.ts
index 6b805d540c254..226a5e741546a 100644
--- a/.buildkite/scripts/pipelines/pull_request/pipeline.ts
+++ b/.buildkite/scripts/pipelines/pull_request/pipeline.ts
@@ -349,6 +349,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
       );
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/discover_enhanced\/ui_tests/,
+        /^packages\/kbn-scout/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:scout-tests')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     // remove duplicated steps
diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
new file mode 100755
index 0000000000000..e58cd40fccbd2
--- /dev/null
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-scout-ui-tests
+
+node scripts/scout_test \
+  --stateful \
+  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
\ No newline at end of file

From 15db0aabcaa1075fdcd8363262513f1d473f36e0 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 3 Dec 2024 15:35:18 +0100
Subject: [PATCH 03/13] refactor and add serverless tests

---
 .../steps/functional/scout_ui_tests.sh        | 19 +++++++++++++++++++
 .../playwright/fixtures/test/validate_tags.ts |  2 +-
 packages/kbn-scout/src/playwright/index.ts    |  2 +-
 .../src/playwright/runner/run_tests.ts        |  9 +++------
 .../src/playwright/{constants.ts => tags.ts}  |  0
 .../kbn-scout/src/playwright/utils/index.ts   |  9 +++++++++
 6 files changed, 33 insertions(+), 8 deletions(-)
 rename packages/kbn-scout/src/playwright/{constants.ts => tags.ts} (100%)

diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
index e58cd40fccbd2..9162402e1d374 100755
--- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@@ -6,7 +6,26 @@ source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-scout-ui-tests
 
+echo "--- Stateful: 'discover_enhanced' plugin UI Tests"
 node scripts/scout_test \
   --stateful \
   --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+
+echo "--- Serverless Elasticsearch: 'discover_enhanced' plugin UI Tests"
+node scripts/scout_test \
+  --serverless=es \
+  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+
+echo "--- Serverless Observability: 'discover_enhanced' plugin UI Tests"
+node scripts/scout_test \
+  --serverless=oblt \
+  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+
+echo "--- Serverless Security: 'discover_enhanced' plugin UI Tests"
+node scripts/scout_test \
+  --serverless=security \
+  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION"
\ No newline at end of file
diff --git a/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts b/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
index 6db0b2d207cde..e4051ec86a124 100644
--- a/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
+++ b/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
@@ -8,7 +8,7 @@
  */
 
 import { test as base } from '@playwright/test';
-import { tags } from '../../constants';
+import { tags } from '../..';
 
 const supportedTags = tags.DEPLOYMENT_AGNOSTIC;
 
diff --git a/packages/kbn-scout/src/playwright/index.ts b/packages/kbn-scout/src/playwright/index.ts
index c12dbd06444c4..0a82bc0468fce 100644
--- a/packages/kbn-scout/src/playwright/index.ts
+++ b/packages/kbn-scout/src/playwright/index.ts
@@ -27,4 +27,4 @@ export type {
 } from './fixtures';
 
 // use to tag tests
-export { tags } from './constants';
+export { tags } from './tags';
diff --git a/packages/kbn-scout/src/playwright/runner/run_tests.ts b/packages/kbn-scout/src/playwright/runner/run_tests.ts
index 9463e6fa201e5..2b9b0c111b441 100644
--- a/packages/kbn-scout/src/playwright/runner/run_tests.ts
+++ b/packages/kbn-scout/src/playwright/runner/run_tests.ts
@@ -18,17 +18,14 @@ import { loadServersConfig } from '../../config';
 import { silence } from '../../common';
 import { RunTestsOptions } from './flags';
 import { getExtraKbnOpts } from '../../servers/run_kibana_server';
-import { tagsByMode } from '../constants';
+import { getPlaywrightGrepTag } from '../utils';
 
 export async function runTests(log: ToolingLog, options: RunTestsOptions) {
   const runStartTime = Date.now();
   const reportTime = getTimeReporter(log, 'scripts/scout_test');
 
   const config = await loadServersConfig(options.mode, log);
-
-  const playwrightTag = config.get('serverless')
-    ? tagsByMode.serverless[config.get('projectType') as keyof typeof tagsByMode.serverless]
-    : tagsByMode.stateful;
+  const playwrightGrepTag = getPlaywrightGrepTag(config);
   const playwrightConfigPath = options.configPath;
 
   await withProcRunner(log, async (procs) => {
@@ -67,7 +64,7 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
         args: [
           'test',
           `--config=${playwrightConfigPath}`,
-          `--grep=${playwrightTag}`,
+          `--grep=${playwrightGrepTag}`,
           ...(options.headed ? ['--headed'] : []),
         ],
         cwd: resolve(REPO_ROOT),
diff --git a/packages/kbn-scout/src/playwright/constants.ts b/packages/kbn-scout/src/playwright/tags.ts
similarity index 100%
rename from packages/kbn-scout/src/playwright/constants.ts
rename to packages/kbn-scout/src/playwright/tags.ts
diff --git a/packages/kbn-scout/src/playwright/utils/index.ts b/packages/kbn-scout/src/playwright/utils/index.ts
index 4b6fcafcbcfa8..8956c6d7cc18f 100644
--- a/packages/kbn-scout/src/playwright/utils/index.ts
+++ b/packages/kbn-scout/src/playwright/utils/index.ts
@@ -8,6 +8,8 @@
  */
 
 import moment from 'moment';
+import { Config } from '../../config';
+import { tagsByMode } from '../tags';
 
 export const serviceLoadedMsg = (name: string) => `scout service loaded: ${name}`;
 
@@ -18,3 +20,10 @@ export const isValidUTCDate = (date: string): boolean => {
 export function formatTime(date: string, fmt: string = 'MMM D, YYYY @ HH:mm:ss.SSS') {
   return moment.utc(date, fmt).format();
 }
+
+export const getPlaywrightGrepTag = (config: Config): string => {
+  const serversConfig = config.getTestServersConfig();
+  return serversConfig.serverless
+    ? tagsByMode.serverless[serversConfig.projectType!]
+    : tagsByMode.stateful;
+};

From 11648e6e9cac718760a67da05cc71206412ff725 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 3 Dec 2024 17:23:22 +0100
Subject: [PATCH 04/13] exclude security project where viewer has no permission
 to index

---
 .../kbn-scout/src/playwright/fixtures/test/validate_tags.ts | 2 +-
 .../discover_enhanced/ui_tests/tests/saved_searches.spec.ts | 6 ++++--
 .../ui_tests/tests/value_suggestions_non_time_based.spec.ts | 6 ++++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts b/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
index e4051ec86a124..1866935440e1a 100644
--- a/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
+++ b/packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts
@@ -8,7 +8,7 @@
  */
 
 import { test as base } from '@playwright/test';
-import { tags } from '../..';
+import { tags } from '../../tags';
 
 const supportedTags = tags.DEPLOYMENT_AGNOSTIC;
 
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts
index c5983d4a93357..94a0d6b7b0638 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/saved_searches.spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { expect, tags } from '@kbn/scout';
+import { expect } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 import type { ExtendedScoutTestFixtures } from '../fixtures';
 
@@ -31,7 +31,9 @@ const assertDataViewIsSelected = async (page: ExtendedScoutTestFixtures['page'],
     'Incorrect data view is selected'
   ).toHaveText(name);
 
-test.describe('Discover app - saved searches', { tag: tags.DEPLOYMENT_AGNOSTIC }, () => {
+test.describe('Discover app - saved searches', { tag: ['@ess', '@svlSearch', '@svlOblt'] }, () => {
+  // TODO: Update to use an ES archive with an index accessible to 'viewer'
+  // for running this test against the Security serverless project.
   const START_TIME = '2019-04-27T23:56:51.374Z';
   const END_TIME = '2019-08-23T16:18:51.821Z';
   const PANEL_NAME = 'Ecommerce Data';
diff --git a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts
index 27e3e03e11f11..d8035597fa0e4 100644
--- a/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts
+++ b/x-pack/plugins/discover_enhanced/ui_tests/tests/value_suggestions_non_time_based.spec.ts
@@ -5,12 +5,14 @@
  * 2.0.
  */
 
-import { expect, tags } from '@kbn/scout';
+import { expect } from '@kbn/scout';
 import { test, testData, assertionMessages } from '../fixtures';
 
 test.describe(
   'Discover app - value suggestions non-time based',
-  { tag: tags.DEPLOYMENT_AGNOSTIC },
+  { tag: ['@ess', '@svlSearch', '@svlOblt'] },
+  // TODO: Update to use an ES archive with an index accessible to 'viewer'
+  // for running this test against the Security serverless project.
   () => {
     test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
       await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.NO_TIME_FIELD);

From d82f4cdbe263479b6a26cae6e9b590041fed18d5 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 3 Dec 2024 17:25:03 +0100
Subject: [PATCH 05/13] bump timeout to 30 min

---
 .buildkite/pipelines/pull_request/scout_ui_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/pull_request/scout_ui_tests.yml b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
index cdbe3b0b02e42..dab411ceae80e 100644
--- a/.buildkite/pipelines/pull_request/scout_ui_tests.yml
+++ b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
@@ -12,7 +12,7 @@ steps:
       - linting_with_types
       - check_types
       - check_oas_snapshot
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
     retry:
       automatic:
         - exit_status: '-1'

From 2774f7bee5dfc8744526fca235eb705bd8a2e5c0 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 3 Dec 2024 17:29:50 +0100
Subject: [PATCH 06/13] update script

---
 .buildkite/scripts/steps/functional/scout_ui_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
index 9162402e1d374..47ad6022adc23 100755
--- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@@ -28,4 +28,4 @@ echo "--- Serverless Security: 'discover_enhanced' plugin UI Tests"
 node scripts/scout_test \
   --serverless=security \
   --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
\ No newline at end of file
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION"

From 3a966c40ec053c88c2058af1be4e6682a79ec1fa Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Fri, 6 Dec 2024 10:05:09 +0100
Subject: [PATCH 07/13] update config

---
 .../src/config/serverless/serverless.base.config.ts          | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/kbn-scout/src/config/serverless/serverless.base.config.ts b/packages/kbn-scout/src/config/serverless/serverless.base.config.ts
index 8b4852f9c9e62..93aba239cf2e7 100644
--- a/packages/kbn-scout/src/config/serverless/serverless.base.config.ts
+++ b/packages/kbn-scout/src/config/serverless/serverless.base.config.ts
@@ -80,6 +80,11 @@ export const defaultConfig: ScoutLoaderConfig = {
       'xpack.security.authc.realms.jwt.jwt1.order=-98',
       `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${getDockerFileMountPath(JWKS_PATH)}`,
       `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`,
+      'serverless.indices.validate_dot_prefixes=true',
+      // controller cluster-settings
+      `cluster.service.slow_task_logging_threshold=15s`,
+      `cluster.service.slow_task_thread_dump_timeout=5s`,
+      `serverless.search.enable_replicas_for_instant_failover=true`,
     ],
     ssl: true, // SSL is required for SAML realm
   },

From ea67b2d04e7833cc24d10e7eaf36cfa74dfe06db Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Mon, 9 Dec 2024 15:14:59 +0100
Subject: [PATCH 08/13] [kbn-scout] fix svl configs to run Kibana build

---
 .../src/config/serverless/es.serverless.config.ts         | 1 +
 .../src/config/serverless/oblt.serverless.config.ts       | 1 +
 .../src/config/serverless/security.serverless.config.ts   | 1 +
 .../src/config/serverless/serverless.base.config.ts       | 8 ++++++++
 4 files changed, 11 insertions(+)

diff --git a/packages/kbn-scout/src/config/serverless/es.serverless.config.ts b/packages/kbn-scout/src/config/serverless/es.serverless.config.ts
index 89e27b4e877e0..0ae2c7e6f0b3f 100644
--- a/packages/kbn-scout/src/config/serverless/es.serverless.config.ts
+++ b/packages/kbn-scout/src/config/serverless/es.serverless.config.ts
@@ -17,6 +17,7 @@ export const servers: ScoutLoaderConfig = {
     serverArgs: [...defaultConfig.esTestCluster.serverArgs],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=es',
diff --git a/packages/kbn-scout/src/config/serverless/oblt.serverless.config.ts b/packages/kbn-scout/src/config/serverless/oblt.serverless.config.ts
index 3f283f140479e..08eb4d9d7cf55 100644
--- a/packages/kbn-scout/src/config/serverless/oblt.serverless.config.ts
+++ b/packages/kbn-scout/src/config/serverless/oblt.serverless.config.ts
@@ -22,6 +22,7 @@ export const servers: ScoutLoaderConfig = {
     ],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=oblt',
diff --git a/packages/kbn-scout/src/config/serverless/security.serverless.config.ts b/packages/kbn-scout/src/config/serverless/security.serverless.config.ts
index f1fa4f53f8988..289790a9ffeb1 100644
--- a/packages/kbn-scout/src/config/serverless/security.serverless.config.ts
+++ b/packages/kbn-scout/src/config/serverless/security.serverless.config.ts
@@ -20,6 +20,7 @@ export const servers: ScoutLoaderConfig = {
     ],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=security',
diff --git a/packages/kbn-scout/src/config/serverless/serverless.base.config.ts b/packages/kbn-scout/src/config/serverless/serverless.base.config.ts
index 93aba239cf2e7..a20a0c3bbe7a7 100644
--- a/packages/kbn-scout/src/config/serverless/serverless.base.config.ts
+++ b/packages/kbn-scout/src/config/serverless/serverless.base.config.ts
@@ -141,7 +141,15 @@ export const defaultConfig: ScoutLoaderConfig = {
       // This ensures that we register the Security SAML API endpoints.
       // In the real world the SAML config is injected by control plane.
       `--plugin-path=${SAML_IDP_PLUGIN_PATH}`,
+      '--xpack.cloud.base_url=https://fake-cloud.elastic.co',
+      '--xpack.cloud.billing_url=/billing/overview/',
+      '--xpack.cloud.deployments_url=/deployments',
       '--xpack.cloud.id=ftr_fake_cloud_id',
+      '--xpack.cloud.organization_url=/account/',
+      '--xpack.cloud.profile_url=/user/settings/',
+      '--xpack.cloud.projects_url=/projects/',
+      '--xpack.cloud.serverless.project_id=fakeprojectid',
+      '--xpack.cloud.users_and_roles_url=/account/members/',
       // Ensure that SAML is used as the default authentication method whenever a user navigates to Kibana. In other
       // words, Kibana should attempt to authenticate the user using the provider with the lowest order if the Login
       // Selector is disabled (which is how Serverless Kibana is configured). By declaring `cloud-basic` with a higher

From a250a59c4263600a31d377731a4af6d73ee43833 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Mon, 9 Dec 2024 15:36:51 +0100
Subject: [PATCH 09/13] update config type

---
 packages/kbn-scout/src/types/config.d.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/kbn-scout/src/types/config.d.ts b/packages/kbn-scout/src/types/config.d.ts
index 14cd27b47fde2..32883cfefd8b0 100644
--- a/packages/kbn-scout/src/types/config.d.ts
+++ b/packages/kbn-scout/src/types/config.d.ts
@@ -25,9 +25,9 @@ export interface ScoutLoaderConfig {
     ssl: boolean;
   };
   kbnTestServer: {
-    env?: any;
-    buildArgs?: string[];
-    sourceArgs?: string[];
+    env: any;
+    buildArgs: string[];
+    sourceArgs: string[];
     serverArgs: string[];
     useDedicatedTastRunner?: boolean;
   };

From 7fb963936e3acd9e9692c77cbdd55240ae12c717 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Mon, 9 Dec 2024 15:54:52 +0100
Subject: [PATCH 10/13] update ci script

---
 .buildkite/scripts/steps/functional/scout_ui_tests.sh | 8 ++++----
 packages/kbn-scout/src/playwright/runner/run_tests.ts | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
index 47ad6022adc23..85bf3c555f10f 100755
--- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@@ -7,25 +7,25 @@ source .buildkite/scripts/steps/functional/common.sh
 export JOB=kibana-scout-ui-tests
 
 echo "--- Stateful: 'discover_enhanced' plugin UI Tests"
-node scripts/scout_test \
+node scripts/scout run-tests \
   --stateful \
   --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION"
 
 echo "--- Serverless Elasticsearch: 'discover_enhanced' plugin UI Tests"
-node scripts/scout_test \
+node scripts/scout run-tests \
   --serverless=es \
   --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION"
 
 echo "--- Serverless Observability: 'discover_enhanced' plugin UI Tests"
-node scripts/scout_test \
+node scripts/scout run-tests \
   --serverless=oblt \
   --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION"
 
 echo "--- Serverless Security: 'discover_enhanced' plugin UI Tests"
-node scripts/scout_test \
+node scripts/scout run-tests \
   --serverless=security \
   --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION"
diff --git a/packages/kbn-scout/src/playwright/runner/run_tests.ts b/packages/kbn-scout/src/playwright/runner/run_tests.ts
index 2b9b0c111b441..766209f169963 100644
--- a/packages/kbn-scout/src/playwright/runner/run_tests.ts
+++ b/packages/kbn-scout/src/playwright/runner/run_tests.ts
@@ -22,7 +22,7 @@ import { getPlaywrightGrepTag } from '../utils';
 
 export async function runTests(log: ToolingLog, options: RunTestsOptions) {
   const runStartTime = Date.now();
-  const reportTime = getTimeReporter(log, 'scripts/scout_test');
+  const reportTime = getTimeReporter(log, 'scripts/scout run-tests');
 
   const config = await loadServersConfig(options.mode, log);
   const playwrightGrepTag = getPlaywrightGrepTag(config);

From 2b49bffa4d3f3411980aa2f09f5d2f25947328d3 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Mon, 9 Dec 2024 17:18:33 +0100
Subject: [PATCH 11/13] update pipeline script

---
 .../pipelines/pull_request/pipeline.ts        |  2 +-
 .../steps/functional/scout_ui_tests.sh        | 48 ++++++++++---------
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.ts b/.buildkite/scripts/pipelines/pull_request/pipeline.ts
index b94bb76f0ac41..217c29c00d9c6 100644
--- a/.buildkite/scripts/pipelines/pull_request/pipeline.ts
+++ b/.buildkite/scripts/pipelines/pull_request/pipeline.ts
@@ -383,7 +383,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
         /^x-pack\/plugins\/discover_enhanced\/ui_tests/,
         /^packages\/kbn-scout/,
       ])) ||
-      GITHUB_PR_LABELS.includes('ci:scout-tests')
+      GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
     }
diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
index 85bf3c555f10f..06dddde65956c 100755
--- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@@ -6,26 +6,28 @@ source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-scout-ui-tests
 
-echo "--- Stateful: 'discover_enhanced' plugin UI Tests"
-node scripts/scout run-tests \
-  --stateful \
-  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
-
-echo "--- Serverless Elasticsearch: 'discover_enhanced' plugin UI Tests"
-node scripts/scout run-tests \
-  --serverless=es \
-  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
-
-echo "--- Serverless Observability: 'discover_enhanced' plugin UI Tests"
-node scripts/scout run-tests \
-  --serverless=oblt \
-  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
-
-echo "--- Serverless Security: 'discover_enhanced' plugin UI Tests"
-node scripts/scout run-tests \
-  --serverless=security \
-  --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+echo "--- Running 'discover_enhanced' plugin UI Tests"
+
+TEST_CONFIG="x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts"
+KIBANA_DIR="$KIBANA_BUILD_LOCATION"
+
+declare -A TESTS=(
+  ["Stateful"]="--stateful"
+  ["Serverless Elasticsearch"]="--serverless=es"
+  ["Serverless Observability"]="--serverless=oblt"
+  ["Serverless Security"]="--serverless=security"
+)
+
+EXIT_CODE=0
+
+for TEST_NAME in "${!TESTS[@]}"; do
+  echo "--- $TEST_NAME: 'discover_enhanced' plugin UI Tests"
+  if ! node scripts/scout run-tests ${TESTS[$TEST_NAME]} --config "$TEST_CONFIG" --kibana-install-dir "$KIBANA_DIR"; then
+    echo "--- $TEST_NAME: failed"
+    EXIT_CODE=1
+  else
+    echo "--- $TEST_NAME: passed"
+  fi
+done
+
+exit $EXIT_CODE

From a514f7386dbe869aa3b27c2c821e592b3bd581f1 Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Mon, 9 Dec 2024 17:46:39 +0100
Subject: [PATCH 12/13] update pipeline

---
 .buildkite/pipelines/pull_request/scout_ui_tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/pull_request/scout_ui_tests.yml b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
index dab411ceae80e..37ea1567c4f42 100644
--- a/.buildkite/pipelines/pull_request/scout_ui_tests.yml
+++ b/.buildkite/pipelines/pull_request/scout_ui_tests.yml
@@ -11,9 +11,8 @@ steps:
       - linting
       - linting_with_types
       - check_types
-      - check_oas_snapshot
     timeout_in_minutes: 30
     retry:
       automatic:
         - exit_status: '-1'
-          limit: 3
+          limit: 2

From 302ad6792370e46a509dda9b35986d449fb5f01d Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Tue, 10 Dec 2024 09:49:09 +0100
Subject: [PATCH 13/13] improve script

---
 .../scripts/steps/functional/scout_ui_tests.sh      | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
index 06dddde65956c..bee4ec4020882 100755
--- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh
+++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh
@@ -6,8 +6,6 @@ source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-scout-ui-tests
 
-echo "--- Running 'discover_enhanced' plugin UI Tests"
-
 TEST_CONFIG="x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts"
 KIBANA_DIR="$KIBANA_BUILD_LOCATION"
 
@@ -18,15 +16,18 @@ declare -A TESTS=(
   ["Serverless Security"]="--serverless=security"
 )
 
+ORDER=("Stateful" "Serverless Elasticsearch" "Serverless Observability" "Serverless Security")
+
 EXIT_CODE=0
 
-for TEST_NAME in "${!TESTS[@]}"; do
+for TEST_NAME in "${ORDER[@]}"; do
+  RUN_MODE="${TESTS[$TEST_NAME]}"
   echo "--- $TEST_NAME: 'discover_enhanced' plugin UI Tests"
-  if ! node scripts/scout run-tests ${TESTS[$TEST_NAME]} --config "$TEST_CONFIG" --kibana-install-dir "$KIBANA_DIR"; then
-    echo "--- $TEST_NAME: failed"
+  if ! node scripts/scout run-tests "$RUN_MODE" --config "$TEST_CONFIG" --kibana-install-dir "$KIBANA_DIR"; then
+    echo "$TEST_NAME: failed"
     EXIT_CODE=1
   else
-    echo "--- $TEST_NAME: passed"
+    echo "$TEST_NAME: passed"
   fi
 done