[scout] add ci pipeline (elastic#202707)

## Summary

Adding basic pipeline to run scout UI tests for stateful and 3
serverless projects. Few tests were skipped for Security project because
esArchives used in tests ingest index that are restricted for default
project roles.

It is an opt-in pipeline to verify kbn/scout changes does not break the
existing tests. It is only for start and we plan to re-work into a
mature pipeline later.

New opt-in
[pipeline](https://buildkite.com/elastic/kibana-pull-request/builds/258589#0193afc4-bbd6-4200-8c5f-a7e4a8073e1d)
was added
<img width="1659" alt="Screenshot 2024-12-10 at 11 31 35"
src="https://github.com/user-attachments/assets/1c19fa46-4e66-4796-ac6d-c2c96c74fa8e">

.buildkite/pipelines/pull_request/scout_ui_tests.yml

@@ -0,0 +1,18 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
+    label: 'Scout UI Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 2

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -378,6 +378,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
       );
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/discover_enhanced\/ui_tests/,
+        /^packages\/kbn-scout/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     // remove duplicated steps

.buildkite/scripts/steps/functional/scout_ui_tests.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-scout-ui-tests
+
+TEST_CONFIG="x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts"
+KIBANA_DIR="$KIBANA_BUILD_LOCATION"
+
+declare -A TESTS=(
+  ["Stateful"]="--stateful"
+  ["Serverless Elasticsearch"]="--serverless=es"
+  ["Serverless Observability"]="--serverless=oblt"
+  ["Serverless Security"]="--serverless=security"
+)
+
+ORDER=("Stateful" "Serverless Elasticsearch" "Serverless Observability" "Serverless Security")
+
+EXIT_CODE=0
+
+for TEST_NAME in "${ORDER[@]}"; do
+  RUN_MODE="${TESTS[$TEST_NAME]}"
+  echo "--- $TEST_NAME: 'discover_enhanced' plugin UI Tests"
+  if ! node scripts/scout run-tests "$RUN_MODE" --config "$TEST_CONFIG" --kibana-install-dir "$KIBANA_DIR"; then
+    echo "$TEST_NAME: failed"
+    EXIT_CODE=1
+  else
+    echo "$TEST_NAME: passed"
+  fi
+done
+
+exit $EXIT_CODE

packages/kbn-scout/index.ts

@@ -8,7 +8,7 @@
  */
 
 export * as cli from './src/cli';
-export { expect, test, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
+export { expect, test, tags, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
 export type {
   ScoutPage,
   ScoutPlaywrightOptions,

packages/kbn-scout/src/config/serverless/es.serverless.config.ts

@@ -17,6 +17,7 @@ export const servers: ScoutLoaderConfig = {
     serverArgs: [...defaultConfig.esTestCluster.serverArgs],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=es',

packages/kbn-scout/src/config/serverless/oblt.serverless.config.ts

@@ -22,6 +22,7 @@ export const servers: ScoutLoaderConfig = {
     ],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=oblt',

packages/kbn-scout/src/config/serverless/security.serverless.config.ts

@@ -20,6 +20,7 @@ export const servers: ScoutLoaderConfig = {
     ],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=security',

packages/kbn-scout/src/config/serverless/serverless.base.config.ts

@@ -80,6 +80,11 @@ export const defaultConfig: ScoutLoaderConfig = {
       'xpack.security.authc.realms.jwt.jwt1.order=-98',
       `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${getDockerFileMountPath(JWKS_PATH)}`,
       `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`,
+      'serverless.indices.validate_dot_prefixes=true',
+      // controller cluster-settings
+      `cluster.service.slow_task_logging_threshold=15s`,
+      `cluster.service.slow_task_thread_dump_timeout=5s`,
+      `serverless.search.enable_replicas_for_instant_failover=true`,
     ],
     ssl: true, // SSL is required for SAML realm
   },
@@ -136,7 +141,15 @@ export const defaultConfig: ScoutLoaderConfig = {
       // This ensures that we register the Security SAML API endpoints.
       // In the real world the SAML config is injected by control plane.
       `--plugin-path=${SAML_IDP_PLUGIN_PATH}`,
+      '--xpack.cloud.base_url=https://fake-cloud.elastic.co',
+      '--xpack.cloud.billing_url=/billing/overview/',
+      '--xpack.cloud.deployments_url=/deployments',
       '--xpack.cloud.id=ftr_fake_cloud_id',
+      '--xpack.cloud.organization_url=/account/',
+      '--xpack.cloud.profile_url=/user/settings/',
+      '--xpack.cloud.projects_url=/projects/',
+      '--xpack.cloud.serverless.project_id=fakeprojectid',
+      '--xpack.cloud.users_and_roles_url=/account/members/',
       // Ensure that SAML is used as the default authentication method whenever a user navigates to Kibana. In other
       // words, Kibana should attempt to authenticate the user using the provider with the lowest order if the Login
       // Selector is disabled (which is how Serverless Kibana is configured). By declaring `cloud-basic` with a higher

packages/kbn-scout/src/playwright/fixtures/test/index.ts

@@ -11,9 +11,11 @@ import { mergeTests } from '@playwright/test';
 import { browserAuthFixture } from './browser_auth';
 import { scoutPageFixture } from './page';
 import { pageObjectsFixture } from './page_objects';
+import { validateTagsFixture } from './validate_tags';
 
 export const scoutTestFixtures = mergeTests(
   browserAuthFixture,
   scoutPageFixture,
-  pageObjectsFixture
+  pageObjectsFixture,
+  validateTagsFixture
 );

packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { test as base } from '@playwright/test';
+import { tags } from '../../tags';
+
+const supportedTags = tags.DEPLOYMENT_AGNOSTIC;
+
+export const validateTagsFixture = base.extend<{ validateTags: void }>({
+  validateTags: [
+    async ({}, use, testInfo) => {
+      if (testInfo.tags.length === 0) {
+        throw new Error(`At least one tag is required: ${supportedTags.join(', ')}`);
+      }
+
+      const invalidTags = testInfo.tags.filter((tag: string) => !supportedTags.includes(tag));
+      if (invalidTags.length > 0) {
+        throw new Error(
+          `Unsupported tag(s) found in test suite "${testInfo.title}": ${invalidTags.join(
+            ', '
+          )}. ` + `Supported tags are: ${supportedTags.join(', ')}.`
+        );
+      }
+
+      await use();
+    },
+    { auto: true },
+  ],
+});

packages/kbn-scout/src/playwright/index.ts

@@ -25,3 +25,6 @@ export type {
   ScoutPage,
   EsArchiverFixture,
 } from './fixtures';
+
+// use to tag tests
+export { tags } from './tags';

packages/kbn-scout/src/playwright/runner/run_tests.ts

@@ -18,12 +18,14 @@ import { loadServersConfig } from '../../config';
 import { silence } from '../../common';
 import { RunTestsOptions } from './flags';
 import { getExtraKbnOpts } from '../../servers/run_kibana_server';
+import { getPlaywrightGrepTag } from '../utils';
 
 export async function runTests(log: ToolingLog, options: RunTestsOptions) {
   const runStartTime = Date.now();
-  const reportTime = getTimeReporter(log, 'scripts/scout_test');
+  const reportTime = getTimeReporter(log, 'scripts/scout run-tests');
 
   const config = await loadServersConfig(options.mode, log);
+  const playwrightGrepTag = getPlaywrightGrepTag(config);
   const playwrightConfigPath = options.configPath;
 
   await withProcRunner(log, async (procs) => {
@@ -59,7 +61,12 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
       // Running 'npx playwright test --config=${playwrightConfigPath}'
       await procs.run(`playwright`, {
         cmd: resolve(REPO_ROOT, './node_modules/.bin/playwright'),
-        args: ['test', `--config=${playwrightConfigPath}`, ...(options.headed ? ['--headed'] : [])],
+        args: [
+          'test',
+          `--config=${playwrightConfigPath}`,
+          `--grep=${playwrightGrepTag}`,
+          ...(options.headed ? ['--headed'] : []),
+        ],
         cwd: resolve(REPO_ROOT),
         env: {
           ...process.env,

packages/kbn-scout/src/playwright/tags.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+const SERVERLESS_ONLY = ['@svlSecurity', '@svlOblt', '@svlSearch'];
+const ESS_ONLY = ['@ess'];
+const DEPLOYMENT_AGNOSTIC = SERVERLESS_ONLY.concat(ESS_ONLY);
+
+export const tags = {
+  ESS_ONLY,
+  SERVERLESS_ONLY,
+  DEPLOYMENT_AGNOSTIC,
+};
+
+export const tagsByMode = {
+  stateful: '@ess',
+  serverless: {
+    es: '@svlSearch',
+    oblt: '@svlOblt',
+    security: '@svlSecurity',
+  },
+};

packages/kbn-scout/src/playwright/utils/index.ts

@@ -8,6 +8,8 @@
  */
 
 import moment from 'moment';
+import { Config } from '../../config';
+import { tagsByMode } from '../tags';
 
 export const serviceLoadedMsg = (name: string) => `scout service loaded: ${name}`;
 
@@ -18,3 +20,10 @@ export const isValidUTCDate = (date: string): boolean => {
 export function formatTime(date: string, fmt: string = 'MMM D, YYYY @ HH:mm:ss.SSS') {
   return moment.utc(date, fmt).format();
 }
+
+export const getPlaywrightGrepTag = (config: Config): string => {
+  const serversConfig = config.getTestServersConfig();
+  return serversConfig.serverless
+    ? tagsByMode.serverless[serversConfig.projectType!]
+    : tagsByMode.stateful;
+};

packages/kbn-scout/src/types/config.d.ts

@@ -25,9 +25,9 @@ export interface ScoutLoaderConfig {
     ssl: boolean;
   };
   kbnTestServer: {
-    env?: any;
-    buildArgs?: string[];
-    sourceArgs?: string[];
+    env: any;
+    buildArgs: string[];
+    sourceArgs: string[];
     serverArgs: string[];
     useDedicatedTestRunner?: boolean;
   };

x-pack/plugins/discover_enhanced/ui_tests/tests/error_handling.spec.ts

@@ -5,10 +5,10 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 
-test.describe('Discover app - errors', { tag: ['@ess'] }, () => {
+test.describe('Discover app - errors', { tag: tags.ESS_ONLY }, () => {
   test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
     await kbnClient.savedObjects.clean({ types: ['search', 'index-pattern'] });
     await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);