[8.x] [scout] add ci pipeline (#202707)

.buildkite/pipelines/pull_request/scout_ui_tests.yml

@@ -0,0 +1,18 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
+    label: 'Scout UI Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 2

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -386,6 +386,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
       );
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/discover_enhanced\/ui_tests/,
+        /^packages\/kbn-scout/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     // remove duplicated steps

.buildkite/scripts/steps/functional/scout_ui_tests.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-scout-ui-tests
+
+TEST_CONFIG="x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts"
+KIBANA_DIR="$KIBANA_BUILD_LOCATION"
+
+echo "--- Stateful: 'discover_enhanced' plugin UI Tests"
+node scripts/scout run-tests --stateful --config "$TEST_CONFIG" --kibana-install-dir "$KIBANA_DIR"

packages/kbn-scout/index.ts

@@ -8,7 +8,7 @@
  */
 
 export * as cli from './src/cli';
-export { expect, test, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
+export { expect, test, tags, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
 export type {
   ScoutPage,
   ScoutPlaywrightOptions,

packages/kbn-scout/src/config/serverless/es.serverless.config.ts

@@ -17,6 +17,7 @@ export const servers: ScoutLoaderConfig = {
     serverArgs: [...defaultConfig.esTestCluster.serverArgs],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=es',

packages/kbn-scout/src/config/serverless/oblt.serverless.config.ts

@@ -22,6 +22,7 @@ export const servers: ScoutLoaderConfig = {
     ],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=oblt',

packages/kbn-scout/src/config/serverless/security.serverless.config.ts

@@ -20,6 +20,7 @@ export const servers: ScoutLoaderConfig = {
     ],
   },
   kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,
       '--serverless=security',

packages/kbn-scout/src/config/serverless/serverless.base.config.ts

@@ -80,6 +80,11 @@ export const defaultConfig: ScoutLoaderConfig = {
       'xpack.security.authc.realms.jwt.jwt1.order=-98',
       `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${getDockerFileMountPath(JWKS_PATH)}`,
       `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`,
+      'serverless.indices.validate_dot_prefixes=true',
+      // controller cluster-settings
+      `cluster.service.slow_task_logging_threshold=15s`,
+      `cluster.service.slow_task_thread_dump_timeout=5s`,
+      `serverless.search.enable_replicas_for_instant_failover=true`,
     ],
     ssl: true, // SSL is required for SAML realm
   },
@@ -136,7 +141,15 @@ export const defaultConfig: ScoutLoaderConfig = {
       // This ensures that we register the Security SAML API endpoints.
       // In the real world the SAML config is injected by control plane.
       `--plugin-path=${SAML_IDP_PLUGIN_PATH}`,
+      '--xpack.cloud.base_url=https://fake-cloud.elastic.co',
+      '--xpack.cloud.billing_url=/billing/overview/',
+      '--xpack.cloud.deployments_url=/deployments',
       '--xpack.cloud.id=ftr_fake_cloud_id',
+      '--xpack.cloud.organization_url=/account/',
+      '--xpack.cloud.profile_url=/user/settings/',
+      '--xpack.cloud.projects_url=/projects/',
+      '--xpack.cloud.serverless.project_id=fakeprojectid',
+      '--xpack.cloud.users_and_roles_url=/account/members/',
       // Ensure that SAML is used as the default authentication method whenever a user navigates to Kibana. In other
       // words, Kibana should attempt to authenticate the user using the provider with the lowest order if the Login
       // Selector is disabled (which is how Serverless Kibana is configured). By declaring `cloud-basic` with a higher

packages/kbn-scout/src/playwright/fixtures/test/index.ts

@@ -11,9 +11,11 @@ import { mergeTests } from '@playwright/test';
 import { browserAuthFixture } from './browser_auth';
 import { scoutPageFixture } from './page';
 import { pageObjectsFixture } from './page_objects';
+import { validateTagsFixture } from './validate_tags';
 
 export const scoutTestFixtures = mergeTests(
   browserAuthFixture,
   scoutPageFixture,
-  pageObjectsFixture
+  pageObjectsFixture,
+  validateTagsFixture
 );

packages/kbn-scout/src/playwright/fixtures/test/validate_tags.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { test as base } from '@playwright/test';
+import { tags } from '../../tags';
+
+const supportedTags = tags.DEPLOYMENT_AGNOSTIC;
+
+export const validateTagsFixture = base.extend<{ validateTags: void }>({
+  validateTags: [
+    async ({}, use, testInfo) => {
+      if (testInfo.tags.length === 0) {
+        throw new Error(`At least one tag is required: ${supportedTags.join(', ')}`);
+      }
+
+      const invalidTags = testInfo.tags.filter((tag: string) => !supportedTags.includes(tag));
+      if (invalidTags.length > 0) {
+        throw new Error(
+          `Unsupported tag(s) found in test suite "${testInfo.title}": ${invalidTags.join(
+            ', '
+          )}. ` + `Supported tags are: ${supportedTags.join(', ')}.`
+        );
+      }
+
+      await use();
+    },
+    { auto: true },
+  ],
+});

packages/kbn-scout/src/playwright/index.ts

@@ -25,3 +25,6 @@ export type {
   ScoutPage,
   EsArchiverFixture,
 } from './fixtures';
+
+// use to tag tests
+export { tags } from './tags';

packages/kbn-scout/src/playwright/runner/run_tests.ts

@@ -18,12 +18,14 @@ import { loadServersConfig } from '../../config';
 import { silence } from '../../common';
 import { RunTestsOptions } from './flags';
 import { getExtraKbnOpts } from '../../servers/run_kibana_server';
+import { getPlaywrightGrepTag } from '../utils';
 
 export async function runTests(log: ToolingLog, options: RunTestsOptions) {
   const runStartTime = Date.now();
-  const reportTime = getTimeReporter(log, 'scripts/scout_test');
+  const reportTime = getTimeReporter(log, 'scripts/scout run-tests');
 
   const config = await loadServersConfig(options.mode, log);
+  const playwrightGrepTag = getPlaywrightGrepTag(config);
   const playwrightConfigPath = options.configPath;
 
   await withProcRunner(log, async (procs) => {
@@ -59,7 +61,12 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
       // Running 'npx playwright test --config=${playwrightConfigPath}'
       await procs.run(`playwright`, {
         cmd: resolve(REPO_ROOT, './node_modules/.bin/playwright'),
-        args: ['test', `--config=${playwrightConfigPath}`, ...(options.headed ? ['--headed'] : [])],
+        args: [
+          'test',
+          `--config=${playwrightConfigPath}`,
+          `--grep=${playwrightGrepTag}`,
+          ...(options.headed ? ['--headed'] : []),
+        ],
         cwd: resolve(REPO_ROOT),
         env: {
           ...process.env,

packages/kbn-scout/src/playwright/tags.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+const SERVERLESS_ONLY = ['@svlSecurity', '@svlOblt', '@svlSearch'];
+const ESS_ONLY = ['@ess'];
+const DEPLOYMENT_AGNOSTIC = SERVERLESS_ONLY.concat(ESS_ONLY);
+
+export const tags = {
+  ESS_ONLY,
+  SERVERLESS_ONLY,
+  DEPLOYMENT_AGNOSTIC,
+};
+
+export const tagsByMode = {
+  stateful: '@ess',
+  serverless: {
+    es: '@svlSearch',
+    oblt: '@svlOblt',
+    security: '@svlSecurity',
+  },
+};

packages/kbn-scout/src/playwright/utils/index.ts

@@ -8,6 +8,8 @@
  */
 
 import moment from 'moment';
+import { Config } from '../../config';
+import { tagsByMode } from '../tags';
 
 export const serviceLoadedMsg = (name: string) => `scout service loaded: ${name}`;
 
@@ -18,3 +20,10 @@ export const isValidUTCDate = (date: string): boolean => {
 export function formatTime(date: string, fmt: string = 'MMM D, YYYY @ HH:mm:ss.SSS') {
   return moment.utc(date, fmt).format();
 }
+
+export const getPlaywrightGrepTag = (config: Config): string => {
+  const serversConfig = config.getTestServersConfig();
+  return serversConfig.serverless
+    ? tagsByMode.serverless[serversConfig.projectType!]
+    : tagsByMode.stateful;
+};

packages/kbn-scout/src/types/config.d.ts

@@ -25,9 +25,9 @@ export interface ScoutLoaderConfig {
     ssl: boolean;
   };
   kbnTestServer: {
-    env?: any;
-    buildArgs?: string[];
-    sourceArgs?: string[];
+    env: any;
+    buildArgs: string[];
+    sourceArgs: string[];
     serverArgs: string[];
     useDedicatedTestRunner?: boolean;
   };

x-pack/plugins/discover_enhanced/ui_tests/tests/error_handling.spec.ts

@@ -5,10 +5,10 @@
  * 2.0.
  */
 
-import { expect } from '@kbn/scout';
+import { expect, tags } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 
-test.describe('Discover app - errors', { tag: ['@ess'] }, () => {
+test.describe('Discover app - errors', { tag: tags.ESS_ONLY }, () => {
   test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
     await kbnClient.savedObjects.clean({ types: ['search', 'index-pattern'] });
     await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);

x-pack/plugins/discover_enhanced/ui_tests/tests/saved_search_embeddable.spec.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ScoutWorkerFixtures, expect } from '@kbn/scout';
+import { ScoutWorkerFixtures, expect, tags } from '@kbn/scout';
 import { test, testData } from '../fixtures';
 
 const createSavedSearch = async (
@@ -37,66 +37,62 @@ const createSavedSearch = async (
     ],
   });
 
-test.describe(
-  'Discover app - saved search embeddable',
-  { tag: ['@ess', '@svlSecurity', '@svlOblt', '@svlSearch'] },
-  () => {
-    const SAVED_SEARCH_TITLE = 'TempSearch';
-    const SAVED_SEARCH_ID = '90943e30-9a47-11e8-b64d-95841ca0b247';
-    test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
-      await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);
-      await kbnClient.importExport.load(testData.KBN_ARCHIVES.DASHBOARD_DRILLDOWNS);
-      await uiSettings.set({
-        defaultIndex: testData.DATA_VIEW_ID.LOGSTASH, // TODO: investigate why it is required for `node scripts/playwright_test.js` run
-        'timepicker:timeDefaults': `{ "from": "${testData.LOGSTASH_DEFAULT_START_TIME}", "to": "${testData.LOGSTASH_DEFAULT_END_TIME}"}`,
-      });
+test.describe('Discover app - saved search embeddable', { tag: tags.DEPLOYMENT_AGNOSTIC }, () => {
+  const SAVED_SEARCH_TITLE = 'TempSearch';
+  const SAVED_SEARCH_ID = '90943e30-9a47-11e8-b64d-95841ca0b247';
+  test.beforeAll(async ({ esArchiver, kbnClient, uiSettings }) => {
+    await esArchiver.loadIfNeeded(testData.ES_ARCHIVES.LOGSTASH);
+    await kbnClient.importExport.load(testData.KBN_ARCHIVES.DASHBOARD_DRILLDOWNS);
+    await uiSettings.set({
+      defaultIndex: testData.DATA_VIEW_ID.LOGSTASH, // TODO: investigate why it is required for `node scripts/playwright_test.js` run
+      'timepicker:timeDefaults': `{ "from": "${testData.LOGSTASH_DEFAULT_START_TIME}", "to": "${testData.LOGSTASH_DEFAULT_END_TIME}"}`,
     });
+  });
 
-    test.afterAll(async ({ kbnClient, uiSettings }) => {
-      await uiSettings.unset('defaultIndex', 'timepicker:timeDefaults');
-      await kbnClient.savedObjects.cleanStandardList();
-    });
+  test.afterAll(async ({ kbnClient, uiSettings }) => {
+    await uiSettings.unset('defaultIndex', 'timepicker:timeDefaults');
+    await kbnClient.savedObjects.cleanStandardList();
+  });
 
-    test.beforeEach(async ({ browserAuth, pageObjects }) => {
-      await browserAuth.loginAsPrivilegedUser();
-      await pageObjects.dashboard.goto();
-    });
+  test.beforeEach(async ({ browserAuth, pageObjects }) => {
+    await browserAuth.loginAsPrivilegedUser();
+    await pageObjects.dashboard.goto();
+  });
 
-    test('should allow removing the dashboard panel after the underlying saved search has been deleted', async ({
+  test('should allow removing the dashboard panel after the underlying saved search has been deleted', async ({
+    kbnClient,
+    page,
+    pageObjects,
+  }) => {
+    await pageObjects.dashboard.openNewDashboard();
+    await createSavedSearch(
       kbnClient,
-      page,
-      pageObjects,
-    }) => {
-      await pageObjects.dashboard.openNewDashboard();
-      await createSavedSearch(
-        kbnClient,
-        SAVED_SEARCH_ID,
-        SAVED_SEARCH_TITLE,
-        testData.DATA_VIEW_ID.LOGSTASH
-      );
-      await pageObjects.dashboard.addPanelFromLibrary(SAVED_SEARCH_TITLE);
-      await page.testSubj.locator('savedSearchTotalDocuments').waitFor({
-        state: 'visible',
-      });
+      SAVED_SEARCH_ID,
+      SAVED_SEARCH_TITLE,
+      testData.DATA_VIEW_ID.LOGSTASH
+    );
+    await pageObjects.dashboard.addPanelFromLibrary(SAVED_SEARCH_TITLE);
+    await page.testSubj.locator('savedSearchTotalDocuments').waitFor({
+      state: 'visible',
+    });
 
-      await pageObjects.dashboard.saveDashboard('Dashboard with deleted saved search');
-      await kbnClient.savedObjects.delete({
-        type: 'search',
-        id: SAVED_SEARCH_ID,
-      });
+    await pageObjects.dashboard.saveDashboard('Dashboard with deleted saved search');
+    await kbnClient.savedObjects.delete({
+      type: 'search',
+      id: SAVED_SEARCH_ID,
+    });
 
-      await page.reload();
-      await page.waitForLoadingIndicatorHidden();
-      await expect(
-        page.testSubj.locator('embeddableError'),
-        'Embeddable error should be displayed'
-      ).toBeVisible();
+    await page.reload();
+    await page.waitForLoadingIndicatorHidden();
+    await expect(
+      page.testSubj.locator('embeddableError'),
+      'Embeddable error should be displayed'
+    ).toBeVisible();
 
-      await pageObjects.dashboard.removePanel('embeddableError');
-      await expect(
-        page.testSubj.locator('embeddableError'),
-        'Embeddable error should not be displayed'
-      ).toBeHidden();
-    });
-  }
-);
+    await pageObjects.dashboard.removePanel('embeddableError');
+    await expect(
+      page.testSubj.locator('embeddableError'),
+      'Embeddable error should not be displayed'
+    ).toBeHidden();
+  });
+});