From 00082db5a349b4dac5f9acc16996ae2f0ee5cbe5 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 27 Jul 2023 18:15:08 -0600 Subject: [PATCH 001/182] Fix docker pull output for serverless --- packages/kbn-es/src/utils/docker.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 6552545e10a3..96e022388859 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -319,7 +319,10 @@ export async function runServerlessEsNode( log.info(chalk.bold(`Running Serverless ES node: ${name}`)); log.indent(4, () => log.info(chalk.dim(`docker ${dockerCmd.join(' ')}`))); - const { stdout } = await execa('docker', dockerCmd); + const { stdout } = await execa('docker', dockerCmd, { + // inherit is required to show Docker pull output and Java console output + stdio: ['ignore', 'inherit', 'inherit'], + }); log.indent(4, () => log.info(`${name} is running. From 3fed98ba300023ff5dcae384b69ac64b79ffc177 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 27 Jul 2023 18:15:28 -0600 Subject: [PATCH 002/182] WIP Add esFrom serverless --- packages/kbn-test/src/es/test_es_cluster.ts | 2 ++ x-pack/test_serverless/api_integration/config.base.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 9b2a3b8010be..f780463939ee 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -218,6 +218,8 @@ export function createTestEsCluster< installPath = (await firstNode.installSource(config)).installPath; } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; + } else if (esFrom === 'serverless') { + return await firstNode.runServerless({ basePath }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { diff --git a/x-pack/test_serverless/api_integration/config.base.ts b/x-pack/test_serverless/api_integration/config.base.ts index ab8c4509b215..fd5717a21dfc 100644 --- a/x-pack/test_serverless/api_integration/config.base.ts +++ b/x-pack/test_serverless/api_integration/config.base.ts @@ -16,6 +16,9 @@ export function createTestConfig(options: CreateTestConfigOptions) { return { ...svlSharedConfig.getAll(), services, + esTestCluster: { + from: 'serverless', + }, kbnTestServer: { ...svlSharedConfig.get('kbnTestServer'), serverArgs: [ From 243e1632c9df3bb3a40d9b37f326557246f32e76 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 14:09:57 -0600 Subject: [PATCH 003/182] Move esTestCluster to base shared config --- x-pack/test_serverless/api_integration/config.base.ts | 3 --- x-pack/test_serverless/shared/config.base.ts | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/test_serverless/api_integration/config.base.ts b/x-pack/test_serverless/api_integration/config.base.ts index 975475661f05..96072b4933a9 100644 --- a/x-pack/test_serverless/api_integration/config.base.ts +++ b/x-pack/test_serverless/api_integration/config.base.ts @@ -19,9 +19,6 @@ export function createTestConfig(options: CreateTestConfigOptions) { ...services, ...options.services, }, - esTestCluster: { - from: 'serverless', - }, kbnTestServer: { ...svlSharedConfig.get('kbnTestServer'), serverArgs: [ diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index fa940bb8f3fc..ac9e510914e8 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -22,8 +22,7 @@ export default async () => { servers, esTestCluster: { - license: 'trial', - from: 'snapshot', + from: 'serverless', }, kbnTestServer: { From fb63a14acecedfa4d7001ab4b041c1bae60ddc24 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 14:56:11 -0600 Subject: [PATCH 004/182] Add support for custom ports --- packages/kbn-es/src/cli_commands/docker.ts | 4 ++- .../kbn-es/src/cli_commands/serverless.ts | 4 ++- packages/kbn-es/src/utils/docker.test.ts | 25 +++++++++++++++++++ packages/kbn-es/src/utils/docker.ts | 21 ++++++++++------ packages/kbn-test/src/es/test_es_cluster.ts | 2 +- 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/kbn-es/src/cli_commands/docker.ts b/packages/kbn-es/src/cli_commands/docker.ts index cb5a57731b90..62983ef5fbaf 100644 --- a/packages/kbn-es/src/cli_commands/docker.ts +++ b/packages/kbn-es/src/cli_commands/docker.ts @@ -12,7 +12,7 @@ import { ToolingLog } from '@kbn/tooling-log'; import { getTimeReporter } from '@kbn/ci-stats-reporter'; import { Cluster } from '../cluster'; -import { DOCKER_IMG, DOCKER_REPO, DOCKER_TAG } from '../utils'; +import { DOCKER_IMG, DOCKER_REPO, DOCKER_TAG, DEFAULT_PORT } from '../utils'; import { Command } from './types'; export const docker: Command = { @@ -27,6 +27,7 @@ export const docker: Command = { --tag Image tag of ES to run from ${DOCKER_REPO} [default: ${DOCKER_TAG}] --image Full path to image of ES to run, has precedence over tag. [default: ${DOCKER_IMG}] --password Sets password for elastic user [default: ${password}] + --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] -E Additional key=value settings to pass to Elasticsearch -D Override Docker command @@ -53,6 +54,7 @@ export const docker: Command = { }, string: ['tag', 'image', 'D'], + number: ['port'], default: defaults, }); diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index ab2d6d4b6392..265fbf780c0f 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -12,7 +12,7 @@ import { ToolingLog } from '@kbn/tooling-log'; import { getTimeReporter } from '@kbn/ci-stats-reporter'; import { Cluster } from '../cluster'; -import { SERVERLESS_REPO, SERVERLESS_TAG, SERVERLESS_IMG } from '../utils'; +import { SERVERLESS_REPO, SERVERLESS_TAG, SERVERLESS_IMG, DEFAULT_PORT } from '../utils'; import { Command } from './types'; export const serverless: Command = { @@ -25,6 +25,7 @@ export const serverless: Command = { --tag Image tag of ES Serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}] --image Full path of ES Serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}] --clean Remove existing file system object store before running + --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] -E Additional key=value settings to pass to Elasticsearch Examples: @@ -50,6 +51,7 @@ export const serverless: Command = { string: ['tag', 'image'], boolean: ['clean'], + number: ['port'], default: defaults, }); diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index ef978fe76c40..f7b270e215fd 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -16,6 +16,7 @@ import { resolveDockerCmd, resolveDockerImage, resolveEsArgs, + resolvePort, runDockerContainer, runServerlessCluster, runServerlessEsNode, @@ -103,6 +104,30 @@ describe('resolveDockerImage()', () => { }); }); +describe('resolvePort()', () => { + test('should return default port when no options', () => { + const port = resolvePort({}); + + expect(port).toMatchInlineSnapshot(` + Array [ + "-p", + "127.0.0.1:9200:9200", + ] + `); + }); + + test('should return custom port when passed in options', () => { + const port = resolvePort({ port: 9220 }); + + expect(port).toMatchInlineSnapshot(` + Array [ + "-p", + "127.0.0.1:9220:9220", + ] + `); + }); +}); + describe('verifyDockerInstalled()', () => { test('should call the correct Docker command and log the version', async () => { execa.mockImplementationOnce(() => Promise.resolve({ stdout: 'Docker Version 123' })); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 96e022388859..8e721c402f73 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -20,6 +20,7 @@ import { EsClusterExecOptions } from '../cluster_exec_options'; interface BaseOptions { tag?: string; image?: string; + port?: number; } export interface DockerOptions extends EsClusterExecOptions, BaseOptions { @@ -38,6 +39,7 @@ interface ServerlessEsNodeArgs { params: string[]; } +export const DEFAULT_PORT = 9200; const DOCKER_REGISTRY = 'docker.elastic.co'; const DOCKER_BASE_CMD = [ @@ -53,9 +55,6 @@ const DOCKER_BASE_CMD = [ '--name', 'es01', - '-p', - '127.0.0.1:9200:9200', - '-p', '127.0.0.1:9300:9300', ]; @@ -117,9 +116,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es01', params: [ - '-p', - '127.0.0.1:9200:9200', - '-p', '127.0.0.1:9300:9300', @@ -190,6 +186,15 @@ export function resolveDockerImage({ return defaultImg; } +/** + * Determine the port to bind the Serverless index node or Docker node to + */ +export function resolvePort(options: ServerlessOptions | DockerOptions) { + const port = options.port ?? DEFAULT_PORT; + + return ['-p', `127.0.0.1:${port}:${port}`]; +} + /** * Verify that Docker is installed locally */ @@ -345,12 +350,13 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO const image = getServerlessImage(options); const nodeNames = await Promise.all( - SERVERLESS_NODES.map(async (node) => { + SERVERLESS_NODES.map(async (node, i) => { await runServerlessEsNode(log, { ...node, image, params: node.params.concat( resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), + i === 0 ? resolvePort(options) : [], volumeCmd ), }); @@ -380,6 +386,7 @@ export function resolveDockerCmd(options: DockerOptions) { return DOCKER_BASE_CMD.concat( resolveEsArgs(DEFAULT_DOCKER_ESARGS, options), + resolvePort(options), getDockerImage(options) ); } diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index f780463939ee..9e06b85e5c2f 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -219,7 +219,7 @@ export function createTestEsCluster< } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; } else if (esFrom === 'serverless') { - return await firstNode.runServerless({ basePath }); + return await firstNode.runServerless({ basePath, port }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { From 4d5c1cb3fc4c5a5e0571e8ed3f1722081850b7d3 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 15:10:03 -0600 Subject: [PATCH 005/182] env var for ES to bind to custom port --- packages/kbn-es/src/utils/docker.test.ts | 2 ++ packages/kbn-es/src/utils/docker.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index f7b270e215fd..ba5267cf4a77 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -123,6 +123,8 @@ describe('resolvePort()', () => { Array [ "-p", "127.0.0.1:9220:9220", + "--env", + "http.port=9220", ] `); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 8e721c402f73..8c81b693886e 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -190,9 +190,16 @@ export function resolveDockerImage({ * Determine the port to bind the Serverless index node or Docker node to */ export function resolvePort(options: ServerlessOptions | DockerOptions) { - const port = options.port ?? DEFAULT_PORT; + if (options.port) { + return [ + '-p', + `127.0.0.1:${options.port}:${options.port}`, + '--env', + `http.port=${options.port}`, + ]; + } - return ['-p', `127.0.0.1:${port}:${port}`]; + return ['-p', `127.0.0.1:${DEFAULT_PORT}:${DEFAULT_PORT}`]; } /** From 81f273c510efab721a1c4d83cb5b29becb584db9 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 15:25:52 -0600 Subject: [PATCH 006/182] Fix type check --- packages/kbn-es/src/cli_commands/docker.ts | 1 - packages/kbn-es/src/cli_commands/serverless.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/kbn-es/src/cli_commands/docker.ts b/packages/kbn-es/src/cli_commands/docker.ts index 62983ef5fbaf..2ef4e125d4f0 100644 --- a/packages/kbn-es/src/cli_commands/docker.ts +++ b/packages/kbn-es/src/cli_commands/docker.ts @@ -54,7 +54,6 @@ export const docker: Command = { }, string: ['tag', 'image', 'D'], - number: ['port'], default: defaults, }); diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index 265fbf780c0f..160e434242da 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -51,7 +51,6 @@ export const serverless: Command = { string: ['tag', 'image'], boolean: ['clean'], - number: ['port'], default: defaults, }); From 37bd9c5f6d8ba576257962890099ef6bea924090 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 17:44:11 -0600 Subject: [PATCH 007/182] pass serverless flag through --- packages/kbn-test/src/es/test_es_cluster.ts | 11 ++++++++--- .../src/functional_tests/lib/run_elasticsearch.ts | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 9e06b85e5c2f..b5a10c5fd554 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -143,6 +143,10 @@ export interface CreateTestEsClusterOptions { * this caller to react appropriately. If this is not passed then an uncatchable exception will be thrown */ onEarlyExit?: (msg: string) => void; + /** + * Is this a serverless project + */ + serverless?: boolean; } export function createTestEsCluster< @@ -164,6 +168,7 @@ export function createTestEsCluster< ssl, transportPort, onEarlyExit, + serverless, } = options; const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`; @@ -214,12 +219,12 @@ export function createTestEsCluster< // We only install once using the first node. If the cluster has // multiple nodes, they'll all share the same ESinstallation. const firstNode = this.nodes[0]; - if (esFrom === 'source') { + if (serverless || esFrom === 'serverless') { + return await firstNode.runServerless({ basePath, port }); + } else if (esFrom === 'source') { installPath = (await firstNode.installSource(config)).installPath; } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; - } else if (esFrom === 'serverless') { - return await firstNode.runServerless({ basePath, port }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index 40d4da7d76d7..f701a5640a8e 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -46,6 +46,7 @@ function getEsConfig({ : config.get('servers.elasticsearch.password'); const dataArchive: string | undefined = config.get('esTestCluster.dataArchive'); + const serverless: boolean = config.get('serverless'); return { ssl, @@ -58,6 +59,7 @@ function getEsConfig({ password, dataArchive, ccsConfig, + serverless, }; } @@ -140,6 +142,7 @@ async function startEsNode({ ], transportPort: config.transportPort, onEarlyExit, + serverless: config.serverless, }); await cluster.start(); From 562624c23e783b8ceb5be32b5a9889a111d064a2 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 18:11:44 -0600 Subject: [PATCH 008/182] Graceful shutdown of serverless nodes --- packages/kbn-es/src/cluster.js | 8 +++++++- packages/kbn-es/src/utils/docker.ts | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index b88e4a788fb7..2c69949725cd 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -22,6 +22,7 @@ const { NativeRealm, parseTimeoutToMs, runServerlessCluster, + killServerlessCluster, runDockerContainer, } = require('./utils'); const { createCliError } = require('./errors'); @@ -295,6 +296,10 @@ exports.Cluster = class Cluster { this._stopCalled; + if (this._serverless) { + return await killServerlessCluster(this._log, this._serverlessNodes); + } + if (!this._process || !this._outcome) { throw new Error('ES has not been started'); } @@ -573,7 +578,8 @@ exports.Cluster = class Cluster { throw new Error('ES has already been started'); } - await runServerlessCluster(this._log, options); + this._serverlessNodes = await runServerlessCluster(this._log, options); + this._serverless = true; } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 8c81b693886e..c36fbfccf9d7 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -374,6 +374,14 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO log.success(`Serverless ES cluster running. Stop the cluster: ${chalk.bold(`docker container stop ${nodeNames.join(' ')}`)} `); + + return nodeNames; +} + +export async function killServerlessCluster(log: ToolingLog, nodes: string[]) { + log.info('Stopping Serverless ES cluster.'); + + await execa('docker', ['container', 'stop'].concat(nodes)); } /** From 8dacb9003d864536b3648fe91a8de79899742cc6 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 28 Jul 2023 18:18:30 -0600 Subject: [PATCH 009/182] Add docker login to buildkite serverless --- .buildkite/scripts/steps/functional/serverless_ftr.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 51eee3307e0b..9c0dadebd9a9 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -6,6 +6,9 @@ source .buildkite/scripts/steps/functional/common.sh export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT" +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT + if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then SERVERLESS_CONFIGS=( "x-pack/test_serverless/api_integration/test_suites/search/config.ts" From f6044c03e67f376318075d846ce84fd1553f51b0 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 1 Aug 2023 11:45:36 -0600 Subject: [PATCH 010/182] Docker login for serverless security --- .buildkite/scripts/steps/functional/security_serverless.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/scripts/steps/functional/security_serverless.sh b/.buildkite/scripts/steps/functional/security_serverless.sh index 6271c8b2e823..055e7bf2ba93 100644 --- a/.buildkite/scripts/steps/functional/security_serverless.sh +++ b/.buildkite/scripts/steps/functional/security_serverless.sh @@ -8,6 +8,9 @@ source .buildkite/scripts/steps/functional/common_cypress.sh export JOB=kibana-serverless-security-cypress export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT + echo "--- Security Serverless Cypress" yarn --cwd x-pack/test_serverless/functional/test_suites/security/cypress cypress:run From 19f116c2ee57365ea8d1724e30996846a4ff257a Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 1 Aug 2023 14:19:01 -0600 Subject: [PATCH 011/182] testing: only detach nodes 2 and 3 --- packages/kbn-es/src/utils/docker.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c36fbfccf9d7..54f0c942d45d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - '--detach', + // '--detach', '--net', 'elastic', @@ -130,6 +130,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ + '--detach', '-p', '127.0.0.1:9202:9202', @@ -147,6 +148,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ + '--detach', '-p', '127.0.0.1:9203:9203', From e613a32fe3f38e79e59075a5f412b0a09738d931 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 1 Aug 2023 15:02:11 -0600 Subject: [PATCH 012/182] Increase vm memory --- .buildkite/scripts/steps/functional/security_serverless.sh | 2 ++ .buildkite/scripts/steps/functional/serverless_ftr.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.buildkite/scripts/steps/functional/security_serverless.sh b/.buildkite/scripts/steps/functional/security_serverless.sh index 055e7bf2ba93..becfe304d489 100644 --- a/.buildkite/scripts/steps/functional/security_serverless.sh +++ b/.buildkite/scripts/steps/functional/security_serverless.sh @@ -11,6 +11,8 @@ export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT +sysctl -w vm.max_map_count=262144 + echo "--- Security Serverless Cypress" yarn --cwd x-pack/test_serverless/functional/test_suites/security/cypress cypress:run diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 9c0dadebd9a9..b073bf08d724 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -9,6 +9,8 @@ export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT +sysctl -w vm.max_map_count=262144 + if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then SERVERLESS_CONFIGS=( "x-pack/test_serverless/api_integration/test_suites/search/config.ts" From 6b467f2b4637425ad48de09ef3aac2b89df84217 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 1 Aug 2023 15:37:32 -0600 Subject: [PATCH 013/182] sudo --- .buildkite/scripts/steps/functional/security_serverless.sh | 2 +- .buildkite/scripts/steps/functional/serverless_ftr.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/functional/security_serverless.sh b/.buildkite/scripts/steps/functional/security_serverless.sh index becfe304d489..7b941162b042 100644 --- a/.buildkite/scripts/steps/functional/security_serverless.sh +++ b/.buildkite/scripts/steps/functional/security_serverless.sh @@ -11,7 +11,7 @@ export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT -sysctl -w vm.max_map_count=262144 +sudo sysctl -w vm.max_map_count=262144 echo "--- Security Serverless Cypress" diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index b073bf08d724..755283d2a22d 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -9,7 +9,7 @@ export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT -sysctl -w vm.max_map_count=262144 +sudo sysctl -w vm.max_map_count=262144 if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then SERVERLESS_CONFIGS=( From e9c0ee15e4fd0f8f27fdadee1bfd42e778639af4 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 1 Aug 2023 16:23:12 -0600 Subject: [PATCH 014/182] test bigger node for serverless --- .buildkite/pipelines/pull_request/base.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index cfae5749f801..f744b6277e34 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -59,7 +59,7 @@ steps: - command: SERVERLESS_ENVIRONMENT=observability .buildkite/scripts/steps/functional/serverless_ftr.sh label: 'Serverless Observability Tests' agents: - queue: n2-4-spot + queue: n2-16-spot depends_on: build timeout_in_minutes: 40 soft_fail: @@ -72,7 +72,7 @@ steps: - command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh label: 'Serverless Search Tests' agents: - queue: n2-4-spot + queue: n2-16-spot depends_on: build timeout_in_minutes: 40 soft_fail: @@ -85,7 +85,7 @@ steps: - command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh label: 'Serverless Security Tests' agents: - queue: n2-4-spot + queue: n2-16-spot depends_on: build timeout_in_minutes: 40 soft_fail: @@ -98,7 +98,7 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless.sh label: 'Serverless Security Cypress Tests' agents: - queue: n2-4-spot + queue: n2-16-spot depends_on: build timeout_in_minutes: 40 soft_fail: From cdd1ba72a8202cdc52179bf6ef6841c16933857c Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 1 Aug 2023 16:49:53 -0600 Subject: [PATCH 015/182] Revert "test bigger node for serverless" This reverts commit e9c0ee15e4fd0f8f27fdadee1bfd42e778639af4. --- .buildkite/pipelines/pull_request/base.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index f744b6277e34..cfae5749f801 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -59,7 +59,7 @@ steps: - command: SERVERLESS_ENVIRONMENT=observability .buildkite/scripts/steps/functional/serverless_ftr.sh label: 'Serverless Observability Tests' agents: - queue: n2-16-spot + queue: n2-4-spot depends_on: build timeout_in_minutes: 40 soft_fail: @@ -72,7 +72,7 @@ steps: - command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh label: 'Serverless Search Tests' agents: - queue: n2-16-spot + queue: n2-4-spot depends_on: build timeout_in_minutes: 40 soft_fail: @@ -85,7 +85,7 @@ steps: - command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh label: 'Serverless Security Tests' agents: - queue: n2-16-spot + queue: n2-4-spot depends_on: build timeout_in_minutes: 40 soft_fail: @@ -98,7 +98,7 @@ steps: - command: .buildkite/scripts/steps/functional/security_serverless.sh label: 'Serverless Security Cypress Tests' agents: - queue: n2-16-spot + queue: n2-4-spot depends_on: build timeout_in_minutes: 40 soft_fail: From 84adcfc7dd86df4ddf14c8d1987a6d890254cf87 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 3 Aug 2023 17:00:00 -0600 Subject: [PATCH 016/182] Revert sysctl in sh scripts --- .buildkite/scripts/steps/functional/security_serverless.sh | 2 -- .buildkite/scripts/steps/functional/serverless_ftr.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/.buildkite/scripts/steps/functional/security_serverless.sh b/.buildkite/scripts/steps/functional/security_serverless.sh index 7b941162b042..055e7bf2ba93 100644 --- a/.buildkite/scripts/steps/functional/security_serverless.sh +++ b/.buildkite/scripts/steps/functional/security_serverless.sh @@ -11,8 +11,6 @@ export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT -sudo sysctl -w vm.max_map_count=262144 - echo "--- Security Serverless Cypress" yarn --cwd x-pack/test_serverless/functional/test_suites/security/cypress cypress:run diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 755283d2a22d..9c0dadebd9a9 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -9,8 +9,6 @@ export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT -sudo sysctl -w vm.max_map_count=262144 - if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then SERVERLESS_CONFIGS=( "x-pack/test_serverless/api_integration/test_suites/search/config.ts" From 50cec7162e19f696d2f5b12ddfcec03cd091d7a0 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 3 Aug 2023 17:18:38 -0600 Subject: [PATCH 017/182] Revert "testing: only detach nodes 2 and 3" This reverts commit 19f116c2ee57365ea8d1724e30996846a4ff257a. --- packages/kbn-es/src/utils/docker.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 54f0c942d45d..c36fbfccf9d7 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - // '--detach', + '--detach', '--net', 'elastic', @@ -130,7 +130,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - '--detach', '-p', '127.0.0.1:9202:9202', @@ -148,7 +147,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - '--detach', '-p', '127.0.0.1:9203:9203', From f26c158e355489a0ea9bbc071a3743669d025a80 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 3 Aug 2023 17:52:35 -0600 Subject: [PATCH 018/182] Pull image separate from run command --- packages/kbn-es/src/utils/docker.ts | 49 ++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c36fbfccf9d7..a1809966adb0 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -239,12 +239,42 @@ export async function maybeCreateDockerNetwork(log: ToolingLog) { log.indent(-4); } +/** + * + * Pull a Docker image if needed. + * Stops serverless from pulling the same image in each node's promise and + * gives better control of log output, instead of falling back to docker run. + */ +export async function maybePullDockerImage(log: ToolingLog, image: string) { + log.info(chalk.bold(`Checking for local image: ${image}`)); + log.indent(4); + + const process = await execa('docker', ['pull', image], { + // inherit is required to show Docker output + stdio: ['ignore', 'inherit', 'inherit'], + }); + // .catch(({ message }) => { + // if (message.includes('network with name elastic already exists')) { + // log.info('Using existing network.'); + // } else { + // throw createCliError(message); + // } + // }); + + if (process?.exitCode === 0) { + log.success('Pulled Image.'); + } + + log.indent(-4); +} + /** * Common setup for Docker and Serverless containers */ -async function setupDocker(log: ToolingLog) { +async function setupDocker(log: ToolingLog, image: string) { await verifyDockerInstalled(log); await maybeCreateDockerNetwork(log); + await maybePullDockerImage(log, image); } /** @@ -351,10 +381,10 @@ export async function runServerlessEsNode( * Runs an ES Serverless Cluster through Docker */ export async function runServerlessCluster(log: ToolingLog, options: ServerlessOptions) { - await setupDocker(log); + const image = getServerlessImage(options); + await setupDocker(log, image); const volumeCmd = await setupServerlessVolumes(log, options); - const image = getServerlessImage(options); const nodeNames = await Promise.all( SERVERLESS_NODES.map(async (node, i) => { @@ -394,7 +424,7 @@ function getDockerImage(options: DockerOptions) { /** * Resolve the full command to run Elasticsearch Docker container */ -export function resolveDockerCmd(options: DockerOptions) { +export function resolveDockerCmd(options: DockerOptions, image: string = DOCKER_IMG) { if (options.dockerCmd) { return options.dockerCmd.split(' '); } @@ -402,7 +432,7 @@ export function resolveDockerCmd(options: DockerOptions) { return DOCKER_BASE_CMD.concat( resolveEsArgs(DEFAULT_DOCKER_ESARGS, options), resolvePort(options), - getDockerImage(options) + image ); } @@ -411,9 +441,14 @@ export function resolveDockerCmd(options: DockerOptions) { * Runs an Elasticsearch Docker Container */ export async function runDockerContainer(log: ToolingLog, options: DockerOptions) { - await setupDocker(log); + let image; + + if (!options.dockerCmd) { + image = getDockerImage(options); + await setupDocker(log, image); + } - const dockerCmd = resolveDockerCmd(options); + const dockerCmd = resolveDockerCmd(options, image); log.info(chalk.dim(`docker ${dockerCmd.join(' ')}`)); return await execa('docker', dockerCmd, { From ff5eb5b360bb962d27bdf470a5b6e034423e0f9c Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 3 Aug 2023 17:53:08 -0600 Subject: [PATCH 019/182] Add maybePullDockerImage tests. Fix tests with setupDocker usage --- packages/kbn-es/src/utils/docker.test.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index ba5267cf4a77..86bd48258ef9 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -13,6 +13,7 @@ import { stat } from 'fs/promises'; import { DOCKER_IMG, maybeCreateDockerNetwork, + maybePullDockerImage, resolveDockerCmd, resolveDockerImage, resolveEsArgs, @@ -217,6 +218,17 @@ describe('maybeCreateDockerNetwork()', () => { }); }); +describe('maybePullDockerImage()', () => { + test('should pull the correct image passed', async () => { + execa.mockImplementationOnce(() => Promise.resolve({ exitCode: 0 })); + + await maybePullDockerImage(log, DOCKER_IMG); + + expect(execa.mock.calls[0][0]).toEqual('docker'); + expect(execa.mock.calls[0][1]).toEqual(expect.arrayContaining(['pull', DOCKER_IMG])); + }); +}); + describe('resolveEsArgs()', () => { const defaultEsArgs: Array<[string, string]> = [ ['foo', 'bar'], @@ -360,8 +372,8 @@ describe('runServerlessCluster()', () => { await runServerlessCluster(log, { basePath: baseEsPath }); - // Verify Docker and network then run three nodes - expect(execa.mock.calls).toHaveLength(5); + // setupDocker execa calls then run three nodes + expect(execa.mock.calls).toHaveLength(6); }); }); @@ -391,7 +403,7 @@ describe('runDockerContainer()', () => { execa.mockImplementation(() => Promise.resolve({ stdout: '' })); await expect(runDockerContainer(log, {})).resolves.toEqual({ stdout: '' }); - // Verify Docker and network then run container - expect(execa.mock.calls).toHaveLength(3); + // setupDocker execa calls then run container + expect(execa.mock.calls).toHaveLength(4); }); }); From bb1ff0edb20dfa5065bb2bfbd6fdc9ccfeb18b5a Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 3 Aug 2023 17:53:44 -0600 Subject: [PATCH 020/182] Revert "Fix docker pull output for serverless" This reverts commit 00082db5a349b4dac5f9acc16996ae2f0ee5cbe5. --- packages/kbn-es/src/utils/docker.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index a1809966adb0..c6b317f6073e 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -361,10 +361,7 @@ export async function runServerlessEsNode( log.info(chalk.bold(`Running Serverless ES node: ${name}`)); log.indent(4, () => log.info(chalk.dim(`docker ${dockerCmd.join(' ')}`))); - const { stdout } = await execa('docker', dockerCmd, { - // inherit is required to show Docker pull output and Java console output - stdio: ['ignore', 'inherit', 'inherit'], - }); + const { stdout } = await execa('docker', dockerCmd); log.indent(4, () => log.info(`${name} is running. From eb995e56334722ca4b44e7e26074672b4f178b76 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 3 Aug 2023 18:03:16 -0600 Subject: [PATCH 021/182] Cleanup --- packages/kbn-es/src/utils/docker.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c6b317f6073e..b418ad22242b 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -253,14 +253,8 @@ export async function maybePullDockerImage(log: ToolingLog, image: string) { // inherit is required to show Docker output stdio: ['ignore', 'inherit', 'inherit'], }); - // .catch(({ message }) => { - // if (message.includes('network with name elastic already exists')) { - // log.info('Using existing network.'); - // } else { - // throw createCliError(message); - // } - // }); + // TODO: adjust log output (shows pulled image when already exists), return promise if (process?.exitCode === 0) { log.success('Pulled Image.'); } From fb8ecf07584b511ef7f8a891f7c89839eec33aa5 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 4 Aug 2023 12:45:59 -0600 Subject: [PATCH 022/182] Cleanup --- packages/kbn-es/src/utils/docker.test.ts | 2 +- packages/kbn-es/src/utils/docker.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 86bd48258ef9..4555f7c14895 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -219,7 +219,7 @@ describe('maybeCreateDockerNetwork()', () => { }); describe('maybePullDockerImage()', () => { - test('should pull the correct image passed', async () => { + test('should pull the passed image', async () => { execa.mockImplementationOnce(() => Promise.resolve({ exitCode: 0 })); await maybePullDockerImage(log, DOCKER_IMG); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index b418ad22242b..4122a73b4f2d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -302,7 +302,7 @@ export function resolveEsArgs( export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { const volumePath = resolve(options.basePath, 'stateless'); - log.info(chalk.bold(`Checking for local Serverless ES object store at ${volumePath}`)); + log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); log.indent(4); if (options.clean && fs.existsSync(volumePath)) { @@ -352,7 +352,7 @@ export async function runServerlessEsNode( image ); - log.info(chalk.bold(`Running Serverless ES node: ${name}`)); + log.info(chalk.bold(`Running serverless ES node: ${name}`)); log.indent(4, () => log.info(chalk.dim(`docker ${dockerCmd.join(' ')}`))); const { stdout } = await execa('docker', dockerCmd); @@ -400,7 +400,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO } export async function killServerlessCluster(log: ToolingLog, nodes: string[]) { - log.info('Stopping Serverless ES cluster.'); + log.info('Stopping serverless ES cluster.'); await execa('docker', ['container', 'stop'].concat(nodes)); } @@ -443,7 +443,7 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions log.info(chalk.dim(`docker ${dockerCmd.join(' ')}`)); return await execa('docker', dockerCmd, { - // inherit is required to show Docker pull output and Java console output for pw, enrollment token, etc + // inherit is required to show Docker output and Java console output for pw, enrollment token, etc stdio: ['ignore', 'inherit', 'inherit'], }); } From fd228576f0c9172eb6715f936bcc56b06316efd0 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 4 Aug 2023 12:58:56 -0600 Subject: [PATCH 023/182] Error handling for maybePull --- packages/kbn-es/src/utils/docker.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 4122a73b4f2d..bda34ea15572 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -241,25 +241,19 @@ export async function maybeCreateDockerNetwork(log: ToolingLog) { /** * - * Pull a Docker image if needed. + * Pull a Docker image if needed. Ensures latest image. * Stops serverless from pulling the same image in each node's promise and * gives better control of log output, instead of falling back to docker run. */ export async function maybePullDockerImage(log: ToolingLog, image: string) { - log.info(chalk.bold(`Checking for local image: ${image}`)); - log.indent(4); + log.info(chalk.bold(`Checking for image: ${image}`)); - const process = await execa('docker', ['pull', image], { + await execa('docker', ['pull', image], { // inherit is required to show Docker output stdio: ['ignore', 'inherit', 'inherit'], + }).catch(({ message }) => { + throw createCliError(message); }); - - // TODO: adjust log output (shows pulled image when already exists), return promise - if (process?.exitCode === 0) { - log.success('Pulled Image.'); - } - - log.indent(-4); } /** From cd422d82d6e93fdc7a9b87ef0608af8c90b55422 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 4 Aug 2023 13:36:21 -0600 Subject: [PATCH 024/182] test detach --- packages/kbn-es/src/utils/docker.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index bda34ea15572..183648d0edaf 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - '--detach', + // '--detach', '--net', 'elastic', @@ -130,6 +130,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ + '--detach', '-p', '127.0.0.1:9202:9202', @@ -147,6 +148,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ + '--detach', '-p', '127.0.0.1:9203:9203', From 0361957b3eeb56338c1fe9b5f6b7678fa2ad77db Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 4 Aug 2023 14:13:40 -0600 Subject: [PATCH 025/182] sleep --- .buildkite/scripts/steps/functional/serverless_ftr.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 9c0dadebd9a9..3964fc8db770 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -40,6 +40,8 @@ do LAST_CODE=$? set -e; + sleep 86400 + if [ $LAST_CODE -ne 0 ]; then EXIT_CODE=10 fi From e1bf85249f88a5a2a0115882705759fdb1dfefcf Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 4 Aug 2023 14:14:02 -0600 Subject: [PATCH 026/182] Revert "test detach" This reverts commit cd422d82d6e93fdc7a9b87ef0608af8c90b55422. --- packages/kbn-es/src/utils/docker.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 183648d0edaf..bda34ea15572 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - // '--detach', + '--detach', '--net', 'elastic', @@ -130,7 +130,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - '--detach', '-p', '127.0.0.1:9202:9202', @@ -148,7 +147,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - '--detach', '-p', '127.0.0.1:9203:9203', From 54881f48783869a7bca88d5dd5a159a10ebaa241 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 4 Aug 2023 15:16:23 -0600 Subject: [PATCH 027/182] Revert "sleep" This reverts commit 0361957b3eeb56338c1fe9b5f6b7678fa2ad77db. --- .buildkite/scripts/steps/functional/serverless_ftr.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 3964fc8db770..9c0dadebd9a9 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -40,8 +40,6 @@ do LAST_CODE=$? set -e; - sleep 86400 - if [ $LAST_CODE -ne 0 ]; then EXIT_CODE=10 fi From 135396392db4029e052fdca24c1e9d87a757306b Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 11:00:45 -0600 Subject: [PATCH 028/182] Use clean object store --- packages/kbn-test/src/es/test_es_cluster.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index b5a10c5fd554..70785b1b3e5f 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -220,7 +220,7 @@ export function createTestEsCluster< // multiple nodes, they'll all share the same ESinstallation. const firstNode = this.nodes[0]; if (serverless || esFrom === 'serverless') { - return await firstNode.runServerless({ basePath, port }); + return await firstNode.runServerless({ basePath, port, clean: true }); } else if (esFrom === 'source') { installPath = (await firstNode.installSource(config)).installPath; } else if (esFrom === 'snapshot') { From f2f50e915f465e2521f45af52cb0714d336bdea3 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 14:46:21 -0600 Subject: [PATCH 029/182] pull docker logs --- packages/kbn-es/src/cluster.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 2c69949725cd..2d826acb9eff 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -580,6 +580,14 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; + + await Promise.all( + this._serverlessNodes.map(async (name) => { + return await execa('docker', ['logs', '-f', name], { + stdio: ['ignore', 'inherit', 'inherit'], + }); + }) + ); } /** From a59bb6fab5d06a2f04cd40800536a08687a43818 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 15:52:52 -0600 Subject: [PATCH 030/182] adjust log style for serverless --- packages/kbn-es/src/utils/docker.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index bda34ea15572..66f2782cc590 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -110,6 +110,8 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'false'], ['cluster.name', 'stateless'], + + ['ES_LOG_STYLE', 'file'], ]; const SERVERLESS_NODES: Array> = [ From bbae2418f7fb2eb6f0c2cf9e7f7d8783e3fc0472 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 15:56:07 -0600 Subject: [PATCH 031/182] log chmod --- packages/kbn-es/src/utils/docker.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 66f2782cc590..39e20a9d55d6 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -315,9 +315,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles } // Permissions are set separately from mkdir due to default umask - await Fsp.chmod(volumePath, 0o766).then(() => - log.info('Setup object store permissions (chmod 766).') - ); + await Fsp.chmod(volumePath, 0o766).then((msg: any) => { + if (msg) { + log.warning(msg); + } + log.info('Setup object store permissions (chmod 766).'); + }); log.indent(-4); From 9e29b45aff3cd20e135193300723706ed9f7ed85 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 16:41:39 -0600 Subject: [PATCH 032/182] adjust base path permissions --- packages/kbn-es/src/utils/docker.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 39e20a9d55d6..60708248ce37 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -314,6 +314,13 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.info('Using existing object store.'); } + await Fsp.chmod(options.basePath, 0o766).then((msg: any) => { + if (msg) { + log.warning(msg); + } + log.info('Setup base path permissions (chmod 766).'); + }); + // Permissions are set separately from mkdir due to default umask await Fsp.chmod(volumePath, 0o766).then((msg: any) => { if (msg) { From bf9f2625a3e133d774e06176a845899ff4706e25 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 16:42:16 -0600 Subject: [PATCH 033/182] change to 777 --- packages/kbn-es/src/utils/docker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 60708248ce37..75b983da7889 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -314,7 +314,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.info('Using existing object store.'); } - await Fsp.chmod(options.basePath, 0o766).then((msg: any) => { + await Fsp.chmod(options.basePath, 0o777).then((msg: any) => { if (msg) { log.warning(msg); } From 9b77f4a93edef9fbeee5f56098844d536d5ce5d5 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 7 Aug 2023 17:43:57 -0600 Subject: [PATCH 034/182] Turn off logging. Add user perm --- packages/kbn-es/src/cluster.js | 14 +++++++------- packages/kbn-es/src/utils/docker.ts | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 2d826acb9eff..26e13cb97415 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -581,13 +581,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; - await Promise.all( - this._serverlessNodes.map(async (name) => { - return await execa('docker', ['logs', '-f', name], { - stdio: ['ignore', 'inherit', 'inherit'], - }); - }) - ); + // await Promise.all( + // this._serverlessNodes.map(async (name) => { + // return await execa('docker', ['logs', '-f', name], { + // stdio: ['ignore', 'inherit', 'inherit'], + // }); + // }) + // ); } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 75b983da7889..d2707a5ab9f8 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -314,13 +314,6 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.info('Using existing object store.'); } - await Fsp.chmod(options.basePath, 0o777).then((msg: any) => { - if (msg) { - log.warning(msg); - } - log.info('Setup base path permissions (chmod 766).'); - }); - // Permissions are set separately from mkdir due to default umask await Fsp.chmod(volumePath, 0o766).then((msg: any) => { if (msg) { @@ -334,6 +327,13 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return ['--volume', `${options.basePath}:/objectstore:z`]; } +async function setupUserPerm() { + const pU = await execa('id', ['-u']); + const pG = await execa('id', ['-g']); + + return ['-u', `${pU.stdout}:${pG.stdout}`]; +} + /** * Resolve the Serverless ES image based on defaults and CLI options */ @@ -382,6 +382,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO await setupDocker(log, image); const volumeCmd = await setupServerlessVolumes(log, options); + const userCmd = await setupUserPerm(); const nodeNames = await Promise.all( SERVERLESS_NODES.map(async (node, i) => { @@ -391,7 +392,8 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO params: node.params.concat( resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), i === 0 ? resolvePort(options) : [], - volumeCmd + volumeCmd, + userCmd ), }); return node.name; From af30185d8f0177d79a86f7322aa560ff62643a07 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 8 Aug 2023 11:20:19 -0600 Subject: [PATCH 035/182] enable logging --- packages/kbn-es/src/cluster.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 26e13cb97415..2d826acb9eff 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -581,13 +581,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; - // await Promise.all( - // this._serverlessNodes.map(async (name) => { - // return await execa('docker', ['logs', '-f', name], { - // stdio: ['ignore', 'inherit', 'inherit'], - // }); - // }) - // ); + await Promise.all( + this._serverlessNodes.map(async (name) => { + return await execa('docker', ['logs', '-f', name], { + stdio: ['ignore', 'inherit', 'inherit'], + }); + }) + ); } /** From 8b6eabc5f9892f415d8100c61aa6d56b968f51e9 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 8 Aug 2023 12:07:27 -0600 Subject: [PATCH 036/182] remove detach --- packages/kbn-es/src/cluster.js | 14 +++++++------- packages/kbn-es/src/utils/docker.ts | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 2d826acb9eff..26e13cb97415 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -581,13 +581,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; - await Promise.all( - this._serverlessNodes.map(async (name) => { - return await execa('docker', ['logs', '-f', name], { - stdio: ['ignore', 'inherit', 'inherit'], - }); - }) - ); + // await Promise.all( + // this._serverlessNodes.map(async (name) => { + // return await execa('docker', ['logs', '-f', name], { + // stdio: ['ignore', 'inherit', 'inherit'], + // }); + // }) + // ); } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index d2707a5ab9f8..2a33b8b35087 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - '--detach', + // '--detach', '--net', 'elastic', @@ -132,6 +132,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ + '--detach', '-p', '127.0.0.1:9202:9202', @@ -149,6 +150,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ + '--detach', '-p', '127.0.0.1:9203:9203', From 45f8154af08eea0b07d1cd88c29e5a850a8eabd2 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 8 Aug 2023 12:48:11 -0600 Subject: [PATCH 037/182] Add privileged. Remove userCmd. Turn on log --- packages/kbn-es/src/cluster.js | 14 +++++++------- packages/kbn-es/src/utils/docker.ts | 14 ++++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 26e13cb97415..2d826acb9eff 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -581,13 +581,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; - // await Promise.all( - // this._serverlessNodes.map(async (name) => { - // return await execa('docker', ['logs', '-f', name], { - // stdio: ['ignore', 'inherit', 'inherit'], - // }); - // }) - // ); + await Promise.all( + this._serverlessNodes.map(async (name) => { + return await execa('docker', ['logs', '-f', name], { + stdio: ['ignore', 'inherit', 'inherit'], + }); + }) + ); } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 2a33b8b35087..cd4dd58148ea 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,9 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - // '--detach', + '--detach', + + '--privileged', '--net', 'elastic', @@ -132,7 +134,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - '--detach', + // '--detach', '-p', '127.0.0.1:9202:9202', @@ -150,7 +152,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - '--detach', + // '--detach', '-p', '127.0.0.1:9203:9203', @@ -384,7 +386,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO await setupDocker(log, image); const volumeCmd = await setupServerlessVolumes(log, options); - const userCmd = await setupUserPerm(); + // const userCmd = await setupUserPerm(); const nodeNames = await Promise.all( SERVERLESS_NODES.map(async (node, i) => { @@ -394,8 +396,8 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO params: node.params.concat( resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), i === 0 ? resolvePort(options) : [], - volumeCmd, - userCmd + volumeCmd + // userCmd ), }); return node.name; From 7b856d718564d2233c91163bece2ebe18bdae4bc Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 8 Aug 2023 13:14:08 -0600 Subject: [PATCH 038/182] Add user namespace host --- packages/kbn-es/src/utils/docker.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index cd4dd58148ea..2451e5986c2a 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -86,6 +86,8 @@ const SHARED_SERVERLESS_PARAMS = [ '--privileged', + '--userns=host', + '--net', 'elastic', @@ -331,12 +333,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return ['--volume', `${options.basePath}:/objectstore:z`]; } -async function setupUserPerm() { - const pU = await execa('id', ['-u']); - const pG = await execa('id', ['-g']); +// async function setupUserPerm() { +// const pU = await execa('id', ['-u']); +// const pG = await execa('id', ['-g']); - return ['-u', `${pU.stdout}:${pG.stdout}`]; -} +// return ['-u', `${pU.stdout}:${pG.stdout}`]; +// } /** * Resolve the Serverless ES image based on defaults and CLI options From 7e370c6e16bb2278e740b4c6691a554d1ff985bf Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 8 Aug 2023 13:47:17 -0600 Subject: [PATCH 039/182] attach --- packages/kbn-es/src/cluster.js | 14 +++++++------- packages/kbn-es/src/utils/docker.ts | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 2d826acb9eff..26e13cb97415 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -581,13 +581,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; - await Promise.all( - this._serverlessNodes.map(async (name) => { - return await execa('docker', ['logs', '-f', name], { - stdio: ['ignore', 'inherit', 'inherit'], - }); - }) - ); + // await Promise.all( + // this._serverlessNodes.map(async (name) => { + // return await execa('docker', ['logs', '-f', name], { + // stdio: ['ignore', 'inherit', 'inherit'], + // }); + // }) + // ); } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 2451e5986c2a..a4b8c8818c0e 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - '--detach', + // '--detach', '--privileged', @@ -136,7 +136,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - // '--detach', + '--detach', '-p', '127.0.0.1:9202:9202', @@ -154,7 +154,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - // '--detach', + '--detach', '-p', '127.0.0.1:9203:9203', From f5035d00057203ca6cf3c8d33bf65e6a70c5a9af Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 8 Aug 2023 16:48:51 -0600 Subject: [PATCH 040/182] Z selinux, remove privileged --- packages/kbn-es/src/utils/docker.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index a4b8c8818c0e..30a8eb7d8524 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -84,9 +84,9 @@ const SHARED_SERVERLESS_PARAMS = [ // '--detach', - '--privileged', + // '--privileged', - '--userns=host', + // '--userns=host', '--net', 'elastic', @@ -330,7 +330,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - return ['--volume', `${options.basePath}:/objectstore:z`]; + return ['--volume', `${options.basePath}:/objectstore:Z`]; } // async function setupUserPerm() { From d1c4fb4855c10840d3df22b27575fa50d1a9ad39 Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 9 Aug 2023 16:00:16 -0600 Subject: [PATCH 041/182] z flag. detach --- packages/kbn-es/src/utils/docker.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 30a8eb7d8524..0b28ae576540 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - // '--detach', + '--detach', // '--privileged', @@ -136,7 +136,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - '--detach', + // '--detach', '-p', '127.0.0.1:9202:9202', @@ -154,7 +154,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - '--detach', + // '--detach', '-p', '127.0.0.1:9203:9203', @@ -330,7 +330,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - return ['--volume', `${options.basePath}:/objectstore:Z`]; + return ['--volume', `${options.basePath}:/objectstore:z`]; } // async function setupUserPerm() { From e6b99da5f4afe9e1a766cbf14d10c8837fa1dc78 Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 9 Aug 2023 16:41:32 -0600 Subject: [PATCH 042/182] user namespace --- packages/kbn-es/src/cluster.js | 14 +++++++------- packages/kbn-es/src/utils/docker.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 26e13cb97415..2d826acb9eff 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -581,13 +581,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); this._serverless = true; - // await Promise.all( - // this._serverlessNodes.map(async (name) => { - // return await execa('docker', ['logs', '-f', name], { - // stdio: ['ignore', 'inherit', 'inherit'], - // }); - // }) - // ); + await Promise.all( + this._serverlessNodes.map(async (name) => { + return await execa('docker', ['logs', '-f', name], { + stdio: ['ignore', 'inherit', 'inherit'], + }); + }) + ); } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 0b28ae576540..13d8f28daa5d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -86,7 +86,7 @@ const SHARED_SERVERLESS_PARAMS = [ // '--privileged', - // '--userns=host', + '--userns=host', '--net', 'elastic', From e20600b089f22979bf6f8d4e9f55b50d40bd6fb5 Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 9 Aug 2023 17:09:11 -0600 Subject: [PATCH 043/182] Cleanup stop cluster --- packages/kbn-es/src/cluster.js | 7 +++---- packages/kbn-es/src/utils/docker.ts | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 2d826acb9eff..d6db263bd779 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -22,7 +22,7 @@ const { NativeRealm, parseTimeoutToMs, runServerlessCluster, - killServerlessCluster, + stopServerlessCluster, runDockerContainer, } = require('./utils'); const { createCliError } = require('./errors'); @@ -296,8 +296,8 @@ exports.Cluster = class Cluster { this._stopCalled; - if (this._serverless) { - return await killServerlessCluster(this._log, this._serverlessNodes); + if (this._serverless?.length) { + return await stopServerlessCluster(this._log, this._serverlessNodes); } if (!this._process || !this._outcome) { @@ -579,7 +579,6 @@ exports.Cluster = class Cluster { } this._serverlessNodes = await runServerlessCluster(this._log, options); - this._serverless = true; await Promise.all( this._serverlessNodes.map(async (name) => { diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 13d8f28daa5d..89ba5662cede 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -413,7 +413,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO return nodeNames; } -export async function killServerlessCluster(log: ToolingLog, nodes: string[]) { +export async function stopServerlessCluster(log: ToolingLog, nodes: string[]) { log.info('Stopping serverless ES cluster.'); await execa('docker', ['container', 'stop'].concat(nodes)); From 880c7ef250d11ed3bca6680a698ec32a9e385682 Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 9 Aug 2023 17:16:51 -0600 Subject: [PATCH 044/182] Add test for stopServerlessCluster --- packages/kbn-es/src/utils/docker.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 4555f7c14895..ff9179413372 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -23,6 +23,7 @@ import { runServerlessEsNode, SERVERLESS_IMG, setupServerlessVolumes, + stopServerlessCluster, verifyDockerInstalled, } from './docker'; import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log'; @@ -377,6 +378,20 @@ describe('runServerlessCluster()', () => { }); }); +describe('stopServerlessCluster()', () => { + test('should stop passed in nodes', async () => { + const nodes = ['es01', 'es02', 'es03']; + execa.mockImplementation(() => Promise.resolve({ stdout: '' })); + + await stopServerlessCluster(log, nodes); + + expect(execa.mock.calls[0][0]).toEqual('docker'); + expect(execa.mock.calls[0][1]).toEqual( + expect.arrayContaining(['container', 'stop'].concat(nodes)) + ); + }); +}); + describe('resolveDockerCmd()', () => { test('should return default command when no options', () => { const dockerCmd = resolveDockerCmd({}); From 64918538ad9dff808fc7d75fefb32a9c1f6b708e Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 9 Aug 2023 17:19:49 -0600 Subject: [PATCH 045/182] attach --- packages/kbn-es/src/cluster.js | 14 +++++++------- packages/kbn-es/src/utils/docker.ts | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index d6db263bd779..5b998cde840f 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -580,13 +580,13 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); - await Promise.all( - this._serverlessNodes.map(async (name) => { - return await execa('docker', ['logs', '-f', name], { - stdio: ['ignore', 'inherit', 'inherit'], - }); - }) - ); + // await Promise.all( + // this._serverlessNodes.map(async (name) => { + // return await execa('docker', ['logs', '-f', name], { + // stdio: ['ignore', 'inherit', 'inherit'], + // }); + // }) + // ); } /** diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 89ba5662cede..ebf425da2ebd 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - '--detach', + // '--detach', // '--privileged', @@ -136,7 +136,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - // '--detach', + '--detach', '-p', '127.0.0.1:9202:9202', @@ -154,7 +154,7 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - // '--detach', + '--detach', '-p', '127.0.0.1:9203:9203', From c3eb57e12e392b1ce2602d7f74595562d60bcc81 Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 9 Aug 2023 17:44:43 -0600 Subject: [PATCH 046/182] user cmd --- packages/kbn-es/src/utils/docker.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index ebf425da2ebd..e2ef8bafe22d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -86,7 +86,7 @@ const SHARED_SERVERLESS_PARAMS = [ // '--privileged', - '--userns=host', + // '--userns=host', '--net', 'elastic', @@ -333,12 +333,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return ['--volume', `${options.basePath}:/objectstore:z`]; } -// async function setupUserPerm() { -// const pU = await execa('id', ['-u']); -// const pG = await execa('id', ['-g']); +async function setupUserPerm() { + const pU = await execa('id', ['-u']); + const pG = await execa('id', ['-g']); -// return ['-u', `${pU.stdout}:${pG.stdout}`]; -// } + return ['-u', `${pU.stdout}:${pG.stdout}`]; +} /** * Resolve the Serverless ES image based on defaults and CLI options @@ -388,7 +388,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO await setupDocker(log, image); const volumeCmd = await setupServerlessVolumes(log, options); - // const userCmd = await setupUserPerm(); + const userCmd = await setupUserPerm(); const nodeNames = await Promise.all( SERVERLESS_NODES.map(async (node, i) => { @@ -398,8 +398,8 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO params: node.params.concat( resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), i === 0 ? resolvePort(options) : [], - volumeCmd - // userCmd + volumeCmd, + userCmd ), }); return node.name; From ab108618d6a95ef24370c2c54ad2953809148a23 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 10:53:01 -0600 Subject: [PATCH 047/182] add sleep --- .buildkite/scripts/steps/functional/serverless_ftr.sh | 2 ++ packages/kbn-es/src/utils/docker.ts | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 9c0dadebd9a9..3964fc8db770 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -40,6 +40,8 @@ do LAST_CODE=$? set -e; + sleep 86400 + if [ $LAST_CODE -ne 0 ]; then EXIT_CODE=10 fi diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index e2ef8bafe22d..34e98fe7b945 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -82,7 +82,7 @@ const SHARED_SERVERLESS_PARAMS = [ '--rm', - // '--detach', + '--detach', // '--privileged', @@ -388,7 +388,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO await setupDocker(log, image); const volumeCmd = await setupServerlessVolumes(log, options); - const userCmd = await setupUserPerm(); + // const userCmd = await setupUserPerm(); const nodeNames = await Promise.all( SERVERLESS_NODES.map(async (node, i) => { @@ -398,8 +398,8 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO params: node.params.concat( resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), i === 0 ? resolvePort(options) : [], - volumeCmd, - userCmd + volumeCmd + // userCmd ), }); return node.name; From a53c6acd98df6611fe53ceb82b150b152eb8c7d4 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 13:11:04 -0600 Subject: [PATCH 048/182] chmod 777. temp fix tag --- packages/kbn-es/src/utils/docker.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 34e98fe7b945..c09239459196 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -74,7 +74,7 @@ export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`; export const DOCKER_IMG = `${DOCKER_REPO}:${DOCKER_TAG}`; export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`; -export const SERVERLESS_TAG = 'latest'; +export const SERVERLESS_TAG = 'git-d82e148f6f28'; // 'latest'; export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`; const SHARED_SERVERLESS_PARAMS = [ @@ -321,11 +321,11 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles } // Permissions are set separately from mkdir due to default umask - await Fsp.chmod(volumePath, 0o766).then((msg: any) => { + await Fsp.chmod(volumePath, 0o777).then((msg: any) => { if (msg) { log.warning(msg); } - log.info('Setup object store permissions (chmod 766).'); + log.info('Setup object store permissions (chmod 777).'); }); log.indent(-4); From 42ab9e4edf4811592885017303c4da93a5de221f Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 14:02:07 -0600 Subject: [PATCH 049/182] Cleanup. Maybe fix shutdown --- .../steps/functional/serverless_ftr.sh | 2 -- packages/kbn-es/src/cluster.js | 5 +++++ packages/kbn-es/src/utils/docker.ts | 20 +------------------ 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 3964fc8db770..9c0dadebd9a9 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -40,8 +40,6 @@ do LAST_CODE=$? set -e; - sleep 86400 - if [ $LAST_CODE -ne 0 ]; then EXIT_CODE=10 fi diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 5b998cde840f..d2517a9645ab 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -277,6 +277,10 @@ exports.Cluster = class Cluster { } this._stopCalled = true; + if (this._serverless?.length) { + return await stopServerlessCluster(this._log, this._serverlessNodes); + } + if (!this._process || !this._outcome) { throw new Error('ES has not been started'); } @@ -580,6 +584,7 @@ exports.Cluster = class Cluster { this._serverlessNodes = await runServerlessCluster(this._log, options); + // TODO: keep? // await Promise.all( // this._serverlessNodes.map(async (name) => { // return await execa('docker', ['logs', '-f', name], { diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c09239459196..6aa8db23f8c2 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -84,10 +84,6 @@ const SHARED_SERVERLESS_PARAMS = [ '--detach', - // '--privileged', - - // '--userns=host', - '--net', 'elastic', @@ -136,7 +132,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es02', params: [ - '--detach', '-p', '127.0.0.1:9202:9202', @@ -154,7 +149,6 @@ const SERVERLESS_NODES: Array> = [ { name: 'es03', params: [ - '--detach', '-p', '127.0.0.1:9203:9203', @@ -321,10 +315,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles } // Permissions are set separately from mkdir due to default umask - await Fsp.chmod(volumePath, 0o777).then((msg: any) => { - if (msg) { - log.warning(msg); - } + await Fsp.chmod(volumePath, 0o777).then(() => { log.info('Setup object store permissions (chmod 777).'); }); @@ -333,13 +324,6 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return ['--volume', `${options.basePath}:/objectstore:z`]; } -async function setupUserPerm() { - const pU = await execa('id', ['-u']); - const pG = await execa('id', ['-g']); - - return ['-u', `${pU.stdout}:${pG.stdout}`]; -} - /** * Resolve the Serverless ES image based on defaults and CLI options */ @@ -388,7 +372,6 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO await setupDocker(log, image); const volumeCmd = await setupServerlessVolumes(log, options); - // const userCmd = await setupUserPerm(); const nodeNames = await Promise.all( SERVERLESS_NODES.map(async (node, i) => { @@ -399,7 +382,6 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), i === 0 ? resolvePort(options) : [], volumeCmd - // userCmd ), }); return node.name; From 07f69dc5afefd67fab94a76649da51b6fd9cc4ba Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 14:17:34 -0600 Subject: [PATCH 050/182] Fix incorrect var --- packages/kbn-es/src/cluster.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index d2517a9645ab..4023e5523daf 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -277,7 +277,7 @@ exports.Cluster = class Cluster { } this._stopCalled = true; - if (this._serverless?.length) { + if (this._serverlessNodes?.length) { return await stopServerlessCluster(this._log, this._serverlessNodes); } @@ -300,7 +300,7 @@ exports.Cluster = class Cluster { this._stopCalled; - if (this._serverless?.length) { + if (this._serverlessNodes?.length) { return await stopServerlessCluster(this._log, this._serverlessNodes); } From ca8eb4e686a10fd183f69439be3a8d1a0eb0aa5c Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 21:54:07 -0600 Subject: [PATCH 051/182] Allow serverless esFrom flag --- packages/kbn-test/src/functional_tests/run_tests/flags.test.ts | 2 +- packages/kbn-test/src/functional_tests/run_tests/flags.ts | 2 +- packages/kbn-test/src/functional_tests/start_servers/flags.ts | 2 +- x-pack/test_serverless/api_integration/config.base.ts | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts index dbb8b1e9762e..8477eecbb910 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts @@ -108,7 +108,7 @@ describe('parse runTest flags', () => { it('validates esFrom', () => { expect(() => test({ esFrom: 'foo' })).toThrowErrorMatchingInlineSnapshot( - `"invalid --esFrom, expected one of \\"snapshot\\", \\"source\\""` + `"invalid --esFrom, expected one of \\"snapshot\\", \\"source\\", \\"serverless\\""` ); }); diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.ts index 9f91bf2728cb..bbc0d53c3b59 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.ts @@ -74,7 +74,7 @@ export function parseFlags(flags: FlagsReader) { logsDir: flags.boolean('logToFile') ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) : undefined, - esFrom: flags.enum('esFrom', ['snapshot', 'source']) ?? 'snapshot', + esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']) ?? 'snapshot', installDir: flags.path('kibana-install-dir'), grep: flags.string('grep'), suiteTags: { diff --git a/packages/kbn-test/src/functional_tests/start_servers/flags.ts b/packages/kbn-test/src/functional_tests/start_servers/flags.ts index 0f53ca6866fa..dde6599ea301 100644 --- a/packages/kbn-test/src/functional_tests/start_servers/flags.ts +++ b/packages/kbn-test/src/functional_tests/start_servers/flags.ts @@ -40,7 +40,7 @@ export function parseFlags(flags: FlagsReader) { return { config: configs[0], - esFrom: flags.enum('esFrom', ['source', 'snapshot']), + esFrom: flags.enum('esFrom', ['source', 'snapshot', 'serverless']), esVersion: EsVersion.getDefault(), installDir: flags.string('kibana-install-dir'), logsDir: flags.boolean('logToFile') diff --git a/x-pack/test_serverless/api_integration/config.base.ts b/x-pack/test_serverless/api_integration/config.base.ts index 96072b4933a9..a60591c4008c 100644 --- a/x-pack/test_serverless/api_integration/config.base.ts +++ b/x-pack/test_serverless/api_integration/config.base.ts @@ -15,6 +15,7 @@ export function createTestConfig(options: CreateTestConfigOptions) { return { ...svlSharedConfig.getAll(), + services: { ...services, ...options.services, From 4a01ee1d0bc9ff90b3ead64118c66adbbbe5fe09 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 22:08:29 -0600 Subject: [PATCH 052/182] Fix snapshot overriding config setting --- packages/kbn-test/src/es/test_es_cluster.ts | 7 ++++--- packages/kbn-test/src/functional_tests/run_tests/flags.ts | 4 ++-- .../kbn-test/src/functional_tests/run_tests/run_tests.ts | 3 +++ .../kbn-test/src/functional_tests/start_servers/flags.ts | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 70785b1b3e5f..a248c8d2876d 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -219,12 +219,13 @@ export function createTestEsCluster< // We only install once using the first node. If the cluster has // multiple nodes, they'll all share the same ESinstallation. const firstNode = this.nodes[0]; - if (serverless || esFrom === 'serverless') { - return await firstNode.runServerless({ basePath, port, clean: true }); - } else if (esFrom === 'source') { + if (esFrom === 'source') { installPath = (await firstNode.installSource(config)).installPath; } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; + } else if (esFrom === 'serverless') { + // TODO: pass along config + return await firstNode.runServerless({ basePath, port, clean: true }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.ts index bbc0d53c3b59..f4dd6beb26e8 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.ts @@ -36,7 +36,7 @@ export const FLAG_OPTIONS: FlagOptions = { help: ` --config Define a FTR config that should be executed. Can be specified multiple times --journey Define a Journey that should be executed. Can be specified multiple times - --esFrom Build Elasticsearch from source or run from snapshot. Default: $TEST_ES_FROM or "snapshot" + --esFrom Build Elasticsearch from source or run snapshot or serverless. Default: $TEST_ES_FROM or "snapshot" --include-tag Tags that suites must include to be run, can be included multiple times --exclude-tag Tags that suites must NOT include to be run, can be included multiple times --include Files that must included to be run, can be included multiple times @@ -74,7 +74,7 @@ export function parseFlags(flags: FlagsReader) { logsDir: flags.boolean('logToFile') ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) : undefined, - esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']) ?? 'snapshot', + esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']), installDir: flags.path('kibana-install-dir'), grep: flags.string('grep'), suiteTags: { diff --git a/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts b/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts index b8edfeadbdf0..81c003900119 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts @@ -42,6 +42,9 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) { dryRun: options.dryRun, grep: options.grep, }, + esTestCluster: { + from: options.esFrom, + }, kbnTestServer: { installDir: options.installDir, }, diff --git a/packages/kbn-test/src/functional_tests/start_servers/flags.ts b/packages/kbn-test/src/functional_tests/start_servers/flags.ts index dde6599ea301..ad2a143f6db7 100644 --- a/packages/kbn-test/src/functional_tests/start_servers/flags.ts +++ b/packages/kbn-test/src/functional_tests/start_servers/flags.ts @@ -23,7 +23,7 @@ export const FLAG_OPTIONS: FlagOptions = { help: ` --config Define a FTR config that should be executed. Can be specified multiple times --journey Define a Journey that should be executed. Can be specified multiple times - --esFrom Build Elasticsearch from source or run from snapshot. Default: $TEST_ES_FROM or "snapshot" + --esFrom Build Elasticsearch from source or run snapshot or serverless. Default: $TEST_ES_FROM or "snapshot" --kibana-install-dir Run Kibana from existing install directory instead of from source --logToFile Write the log output from Kibana/ES to files instead of to stdout `, From 73865eaea318ebb89a87ba487f6d7de058f51b12 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 10 Aug 2023 22:24:53 -0600 Subject: [PATCH 053/182] Fix type check --- packages/kbn-test/src/es/test_es_cluster.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index a248c8d2876d..5e3bbffc3d4c 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -168,7 +168,6 @@ export function createTestEsCluster< ssl, transportPort, onEarlyExit, - serverless, } = options; const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`; From 2936e17e1b31073d7e823b56a390c7da061b3f8c Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 09:34:38 -0600 Subject: [PATCH 054/182] Fix volume test --- packages/kbn-es/src/utils/docker.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index ff9179413372..12a878707632 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -65,7 +65,7 @@ const volumeCmdTest = async (volumeCmd: string[]) => { // extract only permission from mode // eslint-disable-next-line no-bitwise - expect((await stat(serverlessObjectStorePath)).mode & 0o777).toBe(0o766); + expect((await stat(serverlessObjectStorePath)).mode & 0o777).toBe(0o777); }; describe('resolveDockerImage()', () => { From 62edf60ba79869e1de3e094a4a8f10133bb8e536 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 09:36:01 -0600 Subject: [PATCH 055/182] Fix flags test --- packages/kbn-test/src/functional_tests/run_tests/flags.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts index 8477eecbb910..0c9dae3a2579 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts @@ -41,7 +41,7 @@ describe('parse runTest flags', () => { /foo, ], "dryRun": false, - "esFrom": "snapshot", + "esFrom": undefined, "esVersion": , "grep": undefined, "installDir": undefined, From 939b77c317f62d4ea651828c9102b0b223b5116b Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 12:34:56 -0600 Subject: [PATCH 056/182] fix latest tag --- packages/kbn-es/src/utils/docker.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 6aa8db23f8c2..4f1310cfac4b 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -74,7 +74,7 @@ export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`; export const DOCKER_IMG = `${DOCKER_REPO}:${DOCKER_TAG}`; export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`; -export const SERVERLESS_TAG = 'git-d82e148f6f28'; // 'latest'; +export const SERVERLESS_TAG = 'latest'; export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`; const SHARED_SERVERLESS_PARAMS = [ @@ -101,6 +101,10 @@ const SHARED_SERVERLESS_PARAMS = [ '--env', 'path.repo=/objectstore', + + // Temp workaround for ES crashing on latest (2023-07-26) + '--env', + 'data_streams.lifecycle_only.mode=true', ]; // only allow certain ES args to be overwrote by options From c88d3b5f453115d72faef9971b6862ef7ec2e217 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 13:48:23 -0600 Subject: [PATCH 057/182] Skip avatar test --- .../test_suites/common/security/navigation/avatar_menu.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts b/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts index b8647d0423e0..727201e1b361 100644 --- a/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts +++ b/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts @@ -11,7 +11,9 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { const svlCommonPage = getPageObject('svlCommonPage'); const svlCommonNavigation = getService('svlCommonNavigation'); - describe('Avatar menu', function () { + // Requires SSL and security enabled on ES Serverless. + // https://github.com/elastic/kibana/issues/162625 + describe.skip('Avatar menu', function () { it('is displayed', async () => { await svlCommonNavigation.navigateToKibanaHome(); await svlCommonPage.assertUserAvatarExists(); From 68527e95e600cbe83220819b58c1dee22f1f8c33 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 14:10:24 -0600 Subject: [PATCH 058/182] Cleanup --- packages/kbn-es/src/cluster.js | 9 --------- packages/kbn-test/src/es/test_es_cluster.ts | 1 - 2 files changed, 10 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 4023e5523daf..cdb3ece0b212 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -583,15 +583,6 @@ exports.Cluster = class Cluster { } this._serverlessNodes = await runServerlessCluster(this._log, options); - - // TODO: keep? - // await Promise.all( - // this._serverlessNodes.map(async (name) => { - // return await execa('docker', ['logs', '-f', name], { - // stdio: ['ignore', 'inherit', 'inherit'], - // }); - // }) - // ); } /** diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 5e3bbffc3d4c..1363d7e8ed4c 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -223,7 +223,6 @@ export function createTestEsCluster< } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; } else if (esFrom === 'serverless') { - // TODO: pass along config return await firstNode.runServerless({ basePath, port, clean: true }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; From 84d0bc04363f45244bbbe1c0f8c8b57e33e71953 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 16:59:13 -0600 Subject: [PATCH 059/182] Pass esArgs --- packages/kbn-test/src/es/test_es_cluster.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 1363d7e8ed4c..467e3b0382f8 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -223,7 +223,7 @@ export function createTestEsCluster< } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; } else if (esFrom === 'serverless') { - return await firstNode.runServerless({ basePath, port, clean: true }); + return await firstNode.runServerless({ basePath, esArgs: customEsArgs, port, clean: true }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { From 7c7af43881a8605403ddc88ff3bf9d0f7791146b Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 17:25:54 -0600 Subject: [PATCH 060/182] align settings with gradlew --- packages/kbn-es/src/utils/docker.ts | 38 +++++++++++++++------ packages/kbn-test/src/es/test_es_cluster.ts | 8 ++++- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 4f1310cfac4b..711664ca2c7f 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -77,6 +77,8 @@ export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearc export const SERVERLESS_TAG = 'latest'; export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`; +// See for default cluster settings +// https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/kotlin/elasticsearch.serverless-run.gradle.kts const SHARED_SERVERLESS_PARAMS = [ 'run', @@ -87,6 +89,9 @@ const SHARED_SERVERLESS_PARAMS = [ '--net', 'elastic', + '--env', + 'path.repo=/objectstore', + '--env', 'cluster.initial_master_nodes=es01,es02,es03', @@ -99,9 +104,6 @@ const SHARED_SERVERLESS_PARAMS = [ '--env', 'stateless.object_store.bucket=stateless', - '--env', - 'path.repo=/objectstore', - // Temp workaround for ES crashing on latest (2023-07-26) '--env', 'data_streams.lifecycle_only.mode=true', @@ -111,11 +113,19 @@ const SHARED_SERVERLESS_PARAMS = [ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['ES_JAVA_OPTS', '-Xms1g -Xmx1g'], - ['xpack.security.enabled', 'false'], + ['ES_LOG_STYLE', 'file'], ['cluster.name', 'stateless'], - ['ES_LOG_STYLE', 'file'], + ['ingest.geoip.downloader.enabled', 'false'], + + ['xpack.ml.enabled', 'true'], + + ['xpack.security.enabled', 'false'], + + ['xpack.watcher.enabled', 'false'], + + ['xpack.security.operator_privileges.enabled', 'true'], ]; const SERVERLESS_NODES: Array> = [ @@ -129,9 +139,13 @@ const SERVERLESS_NODES: Array> = [ 'discovery.seed_hosts=es02,es03', '--env', - 'node.roles=["master","index"]', + 'node.roles=["master","remote_cluster_client","ingest","index"]', + ], + esArgs: [ + ['xpack.searchable.snapshot.shared_cache.size', '16MB'], + + ['xpack.searchable.snapshot.shared_cache.region_size', '256K'], ], - esArgs: [['xpack.searchable.snapshot.shared_cache.size', '1gb']], }, { name: 'es02', @@ -146,9 +160,13 @@ const SERVERLESS_NODES: Array> = [ 'discovery.seed_hosts=es01,es03', '--env', - 'node.roles=["master","search"]', + 'node.roles=["master","remote_cluster_client","search"]', + ], + esArgs: [ + ['xpack.searchable.snapshot.shared_cache.size', '16MB'], + + ['xpack.searchable.snapshot.shared_cache.region_size', '256K'], ], - esArgs: [['xpack.searchable.snapshot.shared_cache.size', '1gb']], }, { name: 'es03', @@ -163,7 +181,7 @@ const SERVERLESS_NODES: Array> = [ 'discovery.seed_hosts=es01,es02', '--env', - 'node.roles=["master"]', + 'node.roles=["master","remote_cluster_client","ml","transform"]', ], }, ]; diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 467e3b0382f8..a9fbcd73e341 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -223,7 +223,13 @@ export function createTestEsCluster< } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; } else if (esFrom === 'serverless') { - return await firstNode.runServerless({ basePath, esArgs: customEsArgs, port, clean: true }); + return await firstNode.runServerless({ + basePath, + esArgs: customEsArgs, + password, + port, + clean: true, + }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { From b3663ec926560c65d9f27a248e846fde15328c31 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 11 Aug 2023 17:26:39 -0600 Subject: [PATCH 061/182] small fix --- packages/kbn-es/src/utils/docker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 711664ca2c7f..c45801d6ff4c 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -123,9 +123,9 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'false'], - ['xpack.watcher.enabled', 'false'], - ['xpack.security.operator_privileges.enabled', 'true'], + + ['xpack.watcher.enabled', 'false'], ]; const SERVERLESS_NODES: Array> = [ From 242e63ee28f539c91c1e577004afd9ed0f48c122 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 15 Aug 2023 09:41:53 -0600 Subject: [PATCH 062/182] Remove es workaround. Add -i -t --- packages/kbn-es/src/utils/docker.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c45801d6ff4c..9f93e7a91291 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -86,6 +86,10 @@ const SHARED_SERVERLESS_PARAMS = [ '--detach', + '--interactive', + + '--tty', + '--net', 'elastic', @@ -103,10 +107,6 @@ const SHARED_SERVERLESS_PARAMS = [ '--env', 'stateless.object_store.bucket=stateless', - - // Temp workaround for ES crashing on latest (2023-07-26) - '--env', - 'data_streams.lifecycle_only.mode=true', ]; // only allow certain ES args to be overwrote by options From 7265db79ac7c3b865419ec117726d1c07e845063 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 15 Aug 2023 16:04:37 -0600 Subject: [PATCH 063/182] Add teardownServerlessClusterSync and tests --- packages/kbn-es/src/cluster.js | 15 +++++++++--- packages/kbn-es/src/utils/docker.test.ts | 26 +++++++++++++++++++++ packages/kbn-es/src/utils/docker.ts | 22 ++++++++++++++++- packages/kbn-test/src/es/test_es_cluster.ts | 1 + 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index cdb3ece0b212..54ad0595b8c7 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -16,14 +16,15 @@ const { Client } = require('@elastic/elasticsearch'); const { downloadSnapshot, installSnapshot, installSource, installArchive } = require('./install'); const { ES_BIN, ES_PLUGIN_BIN, ES_KEYSTORE_BIN } = require('./paths'); const { - log: defaultLog, - parseEsLog, extractConfigFiles, + log: defaultLog, NativeRealm, + parseEsLog, parseTimeoutToMs, + runDockerContainer, runServerlessCluster, stopServerlessCluster, - runDockerContainer, + teardownServerlessClusterSync, } = require('./utils'); const { createCliError } = require('./errors'); const { promisify } = require('util'); @@ -583,6 +584,14 @@ exports.Cluster = class Cluster { } this._serverlessNodes = await runServerlessCluster(this._log, options); + + if (options.teardown) { + /** + * Ideally would be async and an event like beforeExit or SIGINT, + * but those events are not being triggered in FTR child process. + */ + process.on('exit', () => teardownServerlessClusterSync(this._log)); + } } /** diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 12a878707632..a0c8b1ba9c1c 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -24,6 +24,7 @@ import { SERVERLESS_IMG, setupServerlessVolumes, stopServerlessCluster, + teardownServerlessClusterSync, verifyDockerInstalled, } from './docker'; import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log'; @@ -392,6 +393,31 @@ describe('stopServerlessCluster()', () => { }); }); +describe('teardownServerlessClusterSync()', () => { + test('should kill running serverless nodes', () => { + const nodes = ['es01', 'es02', 'es03']; + execa.commandSync.mockImplementation(() => ({ + stdout: nodes.join('\n'), + })); + + teardownServerlessClusterSync(log); + + expect(execa.commandSync.mock.calls).toHaveLength(2); + expect(execa.commandSync.mock.calls[0][0]).toEqual(expect.stringContaining(SERVERLESS_IMG)); + expect(execa.commandSync.mock.calls[1][0]).toEqual(`docker kill ${nodes.join(' ')}`); + }); + + test('should not kill if no serverless nodes', () => { + execa.commandSync.mockImplementation(() => ({ + stdout: '\n', + })); + + teardownServerlessClusterSync(log); + + expect(execa.commandSync.mock.calls).toHaveLength(1); + }); +}); + describe('resolveDockerCmd()', () => { test('should return default command when no options', () => { const dockerCmd = resolveDockerCmd({}); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 9f93e7a91291..3fa391acdfc1 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -30,6 +30,7 @@ export interface DockerOptions extends EsClusterExecOptions, BaseOptions { export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions { clean?: boolean; basePath: string; + teardown?: boolean; } interface ServerlessEsNodeArgs { @@ -417,12 +418,32 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO return nodeNames; } +/** + * Stop a serverless ES cluster by node names + */ export async function stopServerlessCluster(log: ToolingLog, nodes: string[]) { log.info('Stopping serverless ES cluster.'); await execa('docker', ['container', 'stop'].concat(nodes)); } +/** + * Kill any serverless ES nodes which are running. + */ +export function teardownServerlessClusterSync(log: ToolingLog) { + const { stdout } = execa.commandSync( + `docker ps --filter status=running --filter ancestor=${SERVERLESS_IMG} --quiet` + ); + // Filter empty strings + const runningNodes = stdout.split(/\r?\n/).filter((s) => s); + + if (runningNodes.length) { + log.info('Killing running serverless ES nodes.'); + + execa.commandSync(`docker kill ${runningNodes.join(' ')}`); + } +} + /** * Resolve the Elasticsearch image based on defaults and CLI options */ @@ -446,7 +467,6 @@ export function resolveDockerCmd(options: DockerOptions, image: string = DOCKER_ } /** - * * Runs an Elasticsearch Docker Container */ export async function runDockerContainer(log: ToolingLog, options: DockerOptions) { diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index a9fbcd73e341..2f9489a1cfe2 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -229,6 +229,7 @@ export function createTestEsCluster< password, port, clean: true, + teardown: true, }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; From 6d9f3edce38b299b67d657a9d776e7c6bfa48fbc Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 15 Aug 2023 17:00:11 -0600 Subject: [PATCH 064/182] Skip search test requiring auth --- .../functional/test_suites/search/navigation.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/search/navigation.ts b/x-pack/test_serverless/functional/test_suites/search/navigation.ts index ba19e7b75607..8f6ae17d14ba 100644 --- a/x-pack/test_serverless/functional/test_suites/search/navigation.ts +++ b/x-pack/test_serverless/functional/test_suites/search/navigation.ts @@ -20,7 +20,9 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await svlSearchNavigation.navigateToLandingPage(); }); - it('navigate search sidenav & breadcrumbs', async () => { + // Requires SSL and security enabled on ES Serverless. + // https://github.com/elastic/kibana/issues/162625 + it.skip('navigate search sidenav & breadcrumbs', async () => { const expectNoPageReload = await svlCommonNavigation.createNoPageReloadCheck(); // check serverless search side nav exists From 72e022130dbb5c4055728a9643d906ddf197a479 Mon Sep 17 00:00:00 2001 From: Brad White Date: Wed, 16 Aug 2023 17:53:41 -0600 Subject: [PATCH 065/182] WIP ssl working --- packages/kbn-es/src/utils/docker.ts | 41 +++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 3fa391acdfc1..72fe6cead501 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -122,11 +122,31 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.ml.enabled', 'true'], - ['xpack.security.enabled', 'false'], + // ['xpack.security.enabled', 'false'], + + ['xpack.security.enabled', 'true'], + + ['xpack.security.transport.ssl.enabled', 'true'], + + ['xpack.security.transport.ssl.key', '/usr/share/elasticsearch/config/certs/elasticsearch.key'], + + ['xpack.security.transport.ssl.key_passphrase', 'storepass'], + + [ + 'xpack.security.transport.ssl.certificate', + '/usr/share/elasticsearch/config/certs/elasticsearch.crt', + ], + + [ + 'xpack.security.transport.ssl.certificate_authorities', + '/usr/share/elasticsearch/config/certs/ca.crt', + ], + + ['xpack.security.transport.ssl.verification_mode', 'certificate'], - ['xpack.security.operator_privileges.enabled', 'true'], + // ['xpack.security.operator_privileges.enabled', 'true'], - ['xpack.watcher.enabled', 'false'], + // ['xpack.watcher.enabled', 'false'], ]; const SERVERLESS_NODES: Array> = [ @@ -309,7 +329,8 @@ export function resolveEsArgs( } if (options.password) { - esArgs.set('ELASTIC_PASSWORD', options.password); + // esArgs.set('ELASTIC_PASSWORD', options.password); + // esArgs.set('bootstrap.password', options.password); } return Array.from(esArgs).flatMap((e) => ['--env', e.join('=')]); @@ -320,6 +341,9 @@ export function resolveEsArgs( */ export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { const volumePath = resolve(options.basePath, 'stateless'); + const certs = resolve(options.basePath, 'certs'); + const keystore = resolve(options.basePath, 'config', 'elasticsearch.keystore'); + // const users = resolve(options.basePath, 'config', 'elasticsearch.keystore'); log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); log.indent(4); @@ -344,7 +368,14 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - return ['--volume', `${options.basePath}:/objectstore:z`]; + return [ + '--volume', + `${options.basePath}:/objectstore:z`, + '--volume', + `${certs}:/usr/share/elasticsearch/config/certs:z`, + '--volume', + `${keystore}:/usr/share/elasticsearch/config/elasticsearch.keystore`, + ]; } /** From 0d076c68aa9189754c35c1f51490b649a5b54960 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 09:26:45 -0600 Subject: [PATCH 066/182] Remove skips --- .../test_suites/common/security/navigation/avatar_menu.ts | 4 +--- .../functional/test_suites/search/navigation.ts | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts b/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts index 727201e1b361..b8647d0423e0 100644 --- a/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts +++ b/x-pack/test_serverless/functional/test_suites/common/security/navigation/avatar_menu.ts @@ -11,9 +11,7 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { const svlCommonPage = getPageObject('svlCommonPage'); const svlCommonNavigation = getService('svlCommonNavigation'); - // Requires SSL and security enabled on ES Serverless. - // https://github.com/elastic/kibana/issues/162625 - describe.skip('Avatar menu', function () { + describe('Avatar menu', function () { it('is displayed', async () => { await svlCommonNavigation.navigateToKibanaHome(); await svlCommonPage.assertUserAvatarExists(); diff --git a/x-pack/test_serverless/functional/test_suites/search/navigation.ts b/x-pack/test_serverless/functional/test_suites/search/navigation.ts index 8f6ae17d14ba..ba19e7b75607 100644 --- a/x-pack/test_serverless/functional/test_suites/search/navigation.ts +++ b/x-pack/test_serverless/functional/test_suites/search/navigation.ts @@ -20,9 +20,7 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await svlSearchNavigation.navigateToLandingPage(); }); - // Requires SSL and security enabled on ES Serverless. - // https://github.com/elastic/kibana/issues/162625 - it.skip('navigate search sidenav & breadcrumbs', async () => { + it('navigate search sidenav & breadcrumbs', async () => { const expectNoPageReload = await svlCommonNavigation.createNoPageReloadCheck(); // check serverless search side nav exists From 140f02719dd3ee74b3f4d3ad011b4f5cec57ec66 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 13:21:27 -0600 Subject: [PATCH 067/182] WIP operator and service token setup --- packages/kbn-es/src/utils/docker.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 72fe6cead501..7aad86210435 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -144,7 +144,7 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.security.transport.ssl.verification_mode', 'certificate'], - // ['xpack.security.operator_privileges.enabled', 'true'], + ['xpack.security.operator_privileges.enabled', 'true'], // ['xpack.watcher.enabled', 'false'], ]; @@ -342,8 +342,8 @@ export function resolveEsArgs( export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { const volumePath = resolve(options.basePath, 'stateless'); const certs = resolve(options.basePath, 'certs'); - const keystore = resolve(options.basePath, 'config', 'elasticsearch.keystore'); - // const users = resolve(options.basePath, 'config', 'elasticsearch.keystore'); + const operatorUsers = resolve(options.basePath, 'config', 'operator_users.yml'); + const serviceTokens = resolve(options.basePath, 'config', 'service_tokens'); log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); log.indent(4); @@ -374,7 +374,9 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles '--volume', `${certs}:/usr/share/elasticsearch/config/certs:z`, '--volume', - `${keystore}:/usr/share/elasticsearch/config/elasticsearch.keystore`, + `${operatorUsers}:/usr/share/elasticsearch/config/operator_users.yml`, + '--volume', + `${serviceTokens}:/usr/share/elasticsearch/config/service_tokens`, ]; } @@ -462,6 +464,7 @@ export async function stopServerlessCluster(log: ToolingLog, nodes: string[]) { * Kill any serverless ES nodes which are running. */ export function teardownServerlessClusterSync(log: ToolingLog) { + // TODO: this should use the resolved image const { stdout } = execa.commandSync( `docker ps --filter status=running --filter ancestor=${SERVERLESS_IMG} --quiet` ); From ca11529e51543747afbb23d723bff22419170aae Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 13:24:00 -0600 Subject: [PATCH 068/182] Restore password --- packages/kbn-es/src/utils/docker.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 7aad86210435..d11b0b666bde 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -329,8 +329,7 @@ export function resolveEsArgs( } if (options.password) { - // esArgs.set('ELASTIC_PASSWORD', options.password); - // esArgs.set('bootstrap.password', options.password); + esArgs.set('ELASTIC_PASSWORD', options.password); } return Array.from(esArgs).flatMap((e) => ['--env', e.join('=')]); From 831ffd89c735d80e5f984fa4b35766ef5ce25d1e Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 13:44:00 -0600 Subject: [PATCH 069/182] Cleanup paths. Add ess_resources --- .../src/ess_resources/operator_users.yml | 9 ++++ .../kbn-es/src/ess_resources/service_tokens | 1 + packages/kbn-es/src/paths.ts | 7 ++- packages/kbn-es/src/utils/docker.ts | 46 +++++++++---------- 4 files changed, 38 insertions(+), 25 deletions(-) create mode 100644 packages/kbn-es/src/ess_resources/operator_users.yml create mode 100644 packages/kbn-es/src/ess_resources/service_tokens diff --git a/packages/kbn-es/src/ess_resources/operator_users.yml b/packages/kbn-es/src/ess_resources/operator_users.yml new file mode 100644 index 000000000000..6e13581aa2da --- /dev/null +++ b/packages/kbn-es/src/ess_resources/operator_users.yml @@ -0,0 +1,9 @@ +operator: + - usernames: ["elastic-admin"] + realm_type: "file" + auth_type: "realm" + - usernames: [ "elastic/kibana" ] + realm_type: "_service_account" + auth_type: "token" + token_source: "file" + token_names: [ "kibana-dev" ] \ No newline at end of file diff --git a/packages/kbn-es/src/ess_resources/service_tokens b/packages/kbn-es/src/ess_resources/service_tokens new file mode 100644 index 000000000000..34bc7476f177 --- /dev/null +++ b/packages/kbn-es/src/ess_resources/service_tokens @@ -0,0 +1 @@ +elastic/kibana/kibana-dev:$2a$10$mY2RuGROhk56vLNh.Mgwue98BnkdQPlTR.yGh38ao5jhPJobvuBCq \ No newline at end of file diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 1d909f523302..5fd0aa5207ad 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -7,7 +7,7 @@ */ import Os from 'os'; -import Path from 'path'; +import { resolve } from 'path'; function maybeUseBat(bin: string) { return Os.platform().startsWith('win') ? `${bin}.bat` : bin; @@ -15,7 +15,7 @@ function maybeUseBat(bin: string) { const tempDir = Os.tmpdir(); -export const BASE_PATH = Path.resolve(tempDir, 'kbn-es'); +export const BASE_PATH = resolve(tempDir, 'kbn-es'); export const GRADLE_BIN = maybeUseBat('./gradlew'); export const ES_BIN = maybeUseBat('bin/elasticsearch'); @@ -23,3 +23,6 @@ export const ES_PLUGIN_BIN = maybeUseBat('bin/elasticsearch-plugin'); export const ES_CONFIG = 'config/elasticsearch.yml'; export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore'); + +export const ESS_OPERATOR_USERS = resolve(BASE_PATH, 'ess_resources/operator_users.yml'); +export const ESS_SERVICE_TOKENS = resolve(BASE_PATH, 'ess_resources/service_tokens'); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index d11b0b666bde..5a72b72bb77d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -13,9 +13,11 @@ import { resolve } from 'path'; import { ToolingLog } from '@kbn/tooling-log'; import { kibanaPackageJson as pkg } from '@kbn/repo-info'; +import { ES_P12_PASSWORD } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; +import { ESS_OPERATOR_USERS, ESS_SERVICE_TOKENS } from '../paths'; interface BaseOptions { tag?: string; @@ -42,6 +44,7 @@ interface ServerlessEsNodeArgs { export const DEFAULT_PORT = 9200; const DOCKER_REGISTRY = 'docker.elastic.co'; +const ESS_CONFIG_PATH = '/usr/share/elasticsearch/config/'; const DOCKER_BASE_CMD = [ 'run', @@ -122,31 +125,25 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.ml.enabled', 'true'], - // ['xpack.security.enabled', 'false'], + ['xpack.security.enabled', 'false'], +]; +const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'true'], ['xpack.security.transport.ssl.enabled', 'true'], - ['xpack.security.transport.ssl.key', '/usr/share/elasticsearch/config/certs/elasticsearch.key'], + ['xpack.security.transport.ssl.key', `${ESS_CONFIG_PATH}certs/elasticsearch.key`], - ['xpack.security.transport.ssl.key_passphrase', 'storepass'], + ['xpack.security.transport.ssl.key_passphrase', ES_P12_PASSWORD], - [ - 'xpack.security.transport.ssl.certificate', - '/usr/share/elasticsearch/config/certs/elasticsearch.crt', - ], + ['xpack.security.transport.ssl.certificate', `${ESS_CONFIG_PATH}certs/elasticsearch.crt`], - [ - 'xpack.security.transport.ssl.certificate_authorities', - '/usr/share/elasticsearch/config/certs/ca.crt', - ], + ['xpack.security.transport.ssl.certificate_authorities', `${ESS_CONFIG_PATH}certs/ca.crt`], ['xpack.security.transport.ssl.verification_mode', 'certificate'], ['xpack.security.operator_privileges.enabled', 'true'], - - // ['xpack.watcher.enabled', 'false'], ]; const SERVERLESS_NODES: Array> = [ @@ -339,20 +336,20 @@ export function resolveEsArgs( * Setup local volumes for Serverless ES */ export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { - const volumePath = resolve(options.basePath, 'stateless'); - const certs = resolve(options.basePath, 'certs'); - const operatorUsers = resolve(options.basePath, 'config', 'operator_users.yml'); - const serviceTokens = resolve(options.basePath, 'config', 'service_tokens'); + const { basePath, clean } = options; + + const volumePath = resolve(basePath, 'stateless'); + const certs = resolve(basePath, 'certs'); log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); log.indent(4); - if (options.clean && fs.existsSync(volumePath)) { + if (clean && fs.existsSync(volumePath)) { log.info('Cleaning existing object store.'); await Fsp.rm(volumePath, { recursive: true, force: true }); } - if (options.clean || !fs.existsSync(volumePath)) { + if (clean || !fs.existsSync(volumePath)) { await Fsp.mkdir(volumePath, { recursive: true }).then(() => log.info('Created new object store.') ); @@ -369,13 +366,16 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return [ '--volume', - `${options.basePath}:/objectstore:z`, + `${basePath}:/objectstore:z`, + '--volume', - `${certs}:/usr/share/elasticsearch/config/certs:z`, + `${certs}:${ESS_CONFIG_PATH}certs:z`, + '--volume', - `${operatorUsers}:/usr/share/elasticsearch/config/operator_users.yml`, + `${ESS_OPERATOR_USERS}:${ESS_CONFIG_PATH}operator_users.yml`, + '--volume', - `${serviceTokens}:/usr/share/elasticsearch/config/service_tokens`, + `${ESS_SERVICE_TOKENS}:${ESS_CONFIG_PATH}service_tokens`, ]; } From 75cd21e984560b1394242ae82b28c5e5504e2a6c Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 13:51:00 -0600 Subject: [PATCH 070/182] Add readme --- packages/kbn-es/src/ess_resources/README.md | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 packages/kbn-es/src/ess_resources/README.md diff --git a/packages/kbn-es/src/ess_resources/README.md b/packages/kbn-es/src/ess_resources/README.md new file mode 100644 index 000000000000..70fa7a232f4f --- /dev/null +++ b/packages/kbn-es/src/ess_resources/README.md @@ -0,0 +1,26 @@ +This was originally from the [ESS repository](https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/resources/README.service_tokens.md). + +The "service_tokens" file contains this line: +``` +elastic/kibana/kibana-dev:$2a$10$mY2RuGROhk56vLNh.Mgwue98BnkdQPlTR.yGh38ao5jhPJobvuBCq +``` + +That line defines a single service token +- For the `elastic/kibana` service account +- The token is named `kibana-dev` +- The token's secret is hashed using bcrypt (`$2a$`) using `10` rounds + +Although Elasticsearch used PBKDF2_STRETCH by default, the k8s controller +creates tokens using bcrypt, so we mimic that here. + +The hash is not reversible, so this README is here to tell you what the secret is. +The secret value is: `UUUUUULK-* Z4` +That produces an encoded token of: `AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA` +Yes, the secret was specially chosen to produce an encoded value that can be more easily recognised in development. + +If a node is configured to use this `service_tokens` file, then you can authenticate to it with +``` +curl -H "Authorization: Bearer AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA" http://localhost:9200/_security/_authenticate +``` + +The name of the token (`kibana-dev`) is important because the `operator_users.yml` file designates that token as an operator and allows us to seed an ESS cluster with this token. \ No newline at end of file From 48c6a8cf6bfc8450f6d8b7b31b6eac992f3c7ea6 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 16:30:31 -0600 Subject: [PATCH 071/182] Switch to p12 key --- packages/kbn-es/src/paths.ts | 4 ++-- packages/kbn-es/src/utils/docker.ts | 31 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 5fd0aa5207ad..91e1cd5b2a4c 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -24,5 +24,5 @@ export const ES_CONFIG = 'config/elasticsearch.yml'; export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore'); -export const ESS_OPERATOR_USERS = resolve(BASE_PATH, 'ess_resources/operator_users.yml'); -export const ESS_SERVICE_TOKENS = resolve(BASE_PATH, 'ess_resources/service_tokens'); +export const ESS_OPERATOR_USERS_PATH = resolve(__dirname, './ess_resources/operator_users.yml'); +export const ESS_SERVICE_TOKENS_PATH = resolve(__dirname, './ess_resources/service_tokens'); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 5a72b72bb77d..cb7be0982879 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -13,11 +13,11 @@ import { resolve } from 'path'; import { ToolingLog } from '@kbn/tooling-log'; import { kibanaPackageJson as pkg } from '@kbn/repo-info'; -import { ES_P12_PASSWORD } from '@kbn/dev-utils'; +import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_OPERATOR_USERS, ESS_SERVICE_TOKENS } from '../paths'; +import { ESS_OPERATOR_USERS_PATH, ESS_SERVICE_TOKENS_PATH } from '../paths'; interface BaseOptions { tag?: string; @@ -125,21 +125,25 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.ml.enabled', 'true'], - ['xpack.security.enabled', 'false'], -]; + // ['xpack.security.enabled', 'false'], + // ]; -const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ + // const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'true'], - ['xpack.security.transport.ssl.enabled', 'true'], + // ['xpack.security.http.ssl.enabled', 'true'], - ['xpack.security.transport.ssl.key', `${ESS_CONFIG_PATH}certs/elasticsearch.key`], + // ['xpack.security.http.ssl.keystore.path', `${ESS_CONFIG_PATH}certs/elasticsearch.p12`], - ['xpack.security.transport.ssl.key_passphrase', ES_P12_PASSWORD], + // ['xpack.security.http.ssl.keystore.password', ES_P12_PASSWORD], + + // ['xpack.security.http.ssl.verification_mode', 'certificate'], + + ['xpack.security.transport.ssl.enabled', 'true'], - ['xpack.security.transport.ssl.certificate', `${ESS_CONFIG_PATH}certs/elasticsearch.crt`], + ['xpack.security.transport.ssl.keystore.path', `${ESS_CONFIG_PATH}certs/elasticsearch.p12`], - ['xpack.security.transport.ssl.certificate_authorities', `${ESS_CONFIG_PATH}certs/ca.crt`], + ['xpack.security.transport.ssl.keystore.password', ES_P12_PASSWORD], ['xpack.security.transport.ssl.verification_mode', 'certificate'], @@ -339,7 +343,6 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles const { basePath, clean } = options; const volumePath = resolve(basePath, 'stateless'); - const certs = resolve(basePath, 'certs'); log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); log.indent(4); @@ -369,13 +372,13 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles `${basePath}:/objectstore:z`, '--volume', - `${certs}:${ESS_CONFIG_PATH}certs:z`, + `${ES_P12_PATH}:${ESS_CONFIG_PATH}certs/elasticsearch.p12:z`, '--volume', - `${ESS_OPERATOR_USERS}:${ESS_CONFIG_PATH}operator_users.yml`, + `${ESS_OPERATOR_USERS_PATH}:${ESS_CONFIG_PATH}operator_users.yml`, '--volume', - `${ESS_SERVICE_TOKENS}:${ESS_CONFIG_PATH}service_tokens`, + `${ESS_SERVICE_TOKENS_PATH}:${ESS_CONFIG_PATH}service_tokens`, ]; } From c032116147d664b0ef43721af9479e9bd68c84ad Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 16:32:44 -0600 Subject: [PATCH 072/182] Add HTTP SSL --- packages/kbn-es/src/utils/docker.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index cb7be0982879..c2a300e1e517 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -125,19 +125,19 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.ml.enabled', 'true'], - // ['xpack.security.enabled', 'false'], - // ]; + ['xpack.security.enabled', 'false'], +]; - // const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ +const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'true'], - // ['xpack.security.http.ssl.enabled', 'true'], + ['xpack.security.http.ssl.enabled', 'true'], - // ['xpack.security.http.ssl.keystore.path', `${ESS_CONFIG_PATH}certs/elasticsearch.p12`], + ['xpack.security.http.ssl.keystore.path', `${ESS_CONFIG_PATH}certs/elasticsearch.p12`], - // ['xpack.security.http.ssl.keystore.password', ES_P12_PASSWORD], + ['xpack.security.http.ssl.keystore.password', ES_P12_PASSWORD], - // ['xpack.security.http.ssl.verification_mode', 'certificate'], + ['xpack.security.http.ssl.verification_mode', 'certificate'], ['xpack.security.transport.ssl.enabled', 'true'], From 568ae368b886fface98c3f8cc84a7782efd1c79d Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 16:43:02 -0600 Subject: [PATCH 073/182] remove :z on p12 --- packages/kbn-es/src/utils/docker.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c2a300e1e517..2ddf6fe75f66 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -125,10 +125,10 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.ml.enabled', 'true'], - ['xpack.security.enabled', 'false'], -]; + // ['xpack.security.enabled', 'false'], + // ]; -const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ + // const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'true'], ['xpack.security.http.ssl.enabled', 'true'], @@ -372,7 +372,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles `${basePath}:/objectstore:z`, '--volume', - `${ES_P12_PATH}:${ESS_CONFIG_PATH}certs/elasticsearch.p12:z`, + `${ES_P12_PATH}:${ESS_CONFIG_PATH}certs/elasticsearch.p12`, '--volume', `${ESS_OPERATOR_USERS_PATH}:${ESS_CONFIG_PATH}operator_users.yml`, From b3d400de616babf56bec59bda368199b5507b1b9 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 17 Aug 2023 17:37:53 -0600 Subject: [PATCH 074/182] Add SSL flags. Update tests for SSL support. Add Docker support for SSL --- packages/kbn-es/src/cli_commands/docker.ts | 2 + .../kbn-es/src/cli_commands/serverless.ts | 3 +- packages/kbn-es/src/utils/docker.test.ts | 55 +++++++++++++++++++ packages/kbn-es/src/utils/docker.ts | 52 +++++++++++------- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/packages/kbn-es/src/cli_commands/docker.ts b/packages/kbn-es/src/cli_commands/docker.ts index 2ef4e125d4f0..963d4278349b 100644 --- a/packages/kbn-es/src/cli_commands/docker.ts +++ b/packages/kbn-es/src/cli_commands/docker.ts @@ -28,6 +28,7 @@ export const docker: Command = { --image Full path to image of ES to run, has precedence over tag. [default: ${DOCKER_IMG}] --password Sets password for elastic user [default: ${password}] --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] + --ssl Sets up SSL on Elasticsearch -E Additional key=value settings to pass to Elasticsearch -D Override Docker command @@ -54,6 +55,7 @@ export const docker: Command = { }, string: ['tag', 'image', 'D'], + boolean: ['ssl'], default: defaults, }); diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index 160e434242da..0475910d3c74 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -26,6 +26,7 @@ export const serverless: Command = { --image Full path of ES Serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}] --clean Remove existing file system object store before running --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] + --ssl Sets up SSL on Elasticsearch -E Additional key=value settings to pass to Elasticsearch Examples: @@ -50,7 +51,7 @@ export const serverless: Command = { }, string: ['tag', 'image'], - boolean: ['clean'], + boolean: ['clean', 'ssl'], default: defaults, }); diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index a0c8b1ba9c1c..503c5ac78284 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -28,6 +28,8 @@ import { verifyDockerInstalled, } from './docker'; import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log'; +import { ES_P12_PATH } from '@kbn/dev-utils'; +import { ESS_OPERATOR_USERS_PATH, ESS_SERVICE_TOKENS_PATH } from '../paths'; jest.mock('execa'); const execa = jest.requireMock('execa'); @@ -294,6 +296,43 @@ describe('resolveEsArgs()', () => { ] `); }); + + test('should add SSL args and enable security when SSL is passed', () => { + const esArgs = resolveEsArgs([...defaultEsArgs, ['xpack.security.enabled', 'false']], { + ssl: true, + }); + + expect(esArgs).toHaveLength(24); + expect(esArgs).not.toEqual(expect.arrayContaining(['xpack.security.enabled=false'])); + expect(esArgs).toMatchInlineSnapshot(` + Array [ + "--env", + "foo=bar", + "--env", + "qux=zip", + "--env", + "xpack.security.enabled=true", + "--env", + "xpack.security.http.ssl.enabled=true", + "--env", + "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elasticsearch.p12", + "--env", + "xpack.security.http.ssl.keystore.password=storepass", + "--env", + "xpack.security.http.ssl.verification_mode=certificate", + "--env", + "xpack.security.transport.ssl.enabled=true", + "--env", + "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elasticsearch.p12", + "--env", + "xpack.security.transport.ssl.keystore.password=storepass", + "--env", + "xpack.security.transport.ssl.verification_mode=certificate", + "--env", + "xpack.security.operator_privileges.enabled=true", + ] + `); + }); }); describe('setupServerlessVolumes()', () => { @@ -333,6 +372,22 @@ describe('setupServerlessVolumes()', () => { volumeCmdTest(volumeCmd); expect(existsSync(`${serverlessObjectStorePath}/cluster_state/lease`)).toBe(false); }); + + test('should add SSL volumes when ssl is passed', async () => { + mockFs(existingObjectStore); + + const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath, ssl: true }); + + expect(volumeCmd).toHaveLength(8); + expect( + [ + `${baseEsPath}:/objectstore:z`, + ES_P12_PATH, + ESS_OPERATOR_USERS_PATH, + ESS_SERVICE_TOKENS_PATH, + ].every((path) => volumeCmd.some((cmd) => cmd.includes(path))) + ).toBe(true); + }); }); describe('runServerlessEsNode()', () => { diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 2ddf6fe75f66..375b0ab5bb72 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -23,6 +23,7 @@ interface BaseOptions { tag?: string; image?: string; port?: number; + ssl?: boolean; } export interface DockerOptions extends EsClusterExecOptions, BaseOptions { @@ -125,10 +126,10 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [ ['xpack.ml.enabled', 'true'], - // ['xpack.security.enabled', 'false'], - // ]; + ['xpack.security.enabled', 'false'], +]; - // const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ +const DEFAULT_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.enabled', 'true'], ['xpack.security.http.ssl.enabled', 'true'], @@ -318,10 +319,17 @@ export function resolveEsArgs( defaultEsArgs: Array<[string, string]>, options: ServerlessOptions | DockerOptions ) { + const { esArgs: customEsArgs, password, ssl } = options; const esArgs = new Map(defaultEsArgs); - if (options.esArgs) { - const args = typeof options.esArgs === 'string' ? [options.esArgs] : options.esArgs; + if (ssl) { + DEFAULT_SSL_ESARGS.forEach((arg) => { + esArgs.set(arg[0], arg[1]); + }); + } + + if (customEsArgs) { + const args = typeof customEsArgs === 'string' ? [customEsArgs] : customEsArgs; args.forEach((arg) => { const [key, ...value] = arg.split('='); @@ -329,19 +337,22 @@ export function resolveEsArgs( }); } - if (options.password) { - esArgs.set('ELASTIC_PASSWORD', options.password); + if (password) { + esArgs.set('ELASTIC_PASSWORD', password); } return Array.from(esArgs).flatMap((e) => ['--env', e.join('=')]); } +function getESp12Volume() { + return ['--volume', `${ES_P12_PATH}:${ESS_CONFIG_PATH}certs/elasticsearch.p12`]; +} + /** * Setup local volumes for Serverless ES */ export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { - const { basePath, clean } = options; - + const { basePath, clean, ssl } = options; const volumePath = resolve(basePath, 'stateless'); log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); @@ -367,19 +378,19 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - return [ - '--volume', - `${basePath}:/objectstore:z`, + return ['--volume', `${basePath}:/objectstore:z`].concat( + ssl + ? [ + ...getESp12Volume(), - '--volume', - `${ES_P12_PATH}:${ESS_CONFIG_PATH}certs/elasticsearch.p12`, + '--volume', + `${ESS_OPERATOR_USERS_PATH}:${ESS_CONFIG_PATH}operator_users.yml`, - '--volume', - `${ESS_OPERATOR_USERS_PATH}:${ESS_CONFIG_PATH}operator_users.yml`, - - '--volume', - `${ESS_SERVICE_TOKENS_PATH}:${ESS_CONFIG_PATH}service_tokens`, - ]; + '--volume', + `${ESS_SERVICE_TOKENS_PATH}:${ESS_CONFIG_PATH}service_tokens`, + ] + : [] + ); } /** @@ -498,6 +509,7 @@ export function resolveDockerCmd(options: DockerOptions, image: string = DOCKER_ return DOCKER_BASE_CMD.concat( resolveEsArgs(DEFAULT_DOCKER_ESARGS, options), resolvePort(options), + options.ssl ? getESp12Volume() : [], image ); } From e3e0fedbd2f9aa6094ac56e18661298db2e0ab60 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 21 Aug 2023 11:37:59 -0600 Subject: [PATCH 075/182] Add kibana service account token --- packages/kbn-test/index.ts | 1 + packages/kbn-test/src/kbn/index.ts | 1 + packages/kbn-test/src/kbn/users.ts | 10 ++++++++++ x-pack/test_serverless/shared/config.base.ts | 5 ++--- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index f779803d794b..cae824dc92a6 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -38,6 +38,7 @@ export { kibanaTestUser, adminTestUser, systemIndicesSuperuser, + kibanaServiceAccount, } from './src/kbn'; // @internal diff --git a/packages/kbn-test/src/kbn/index.ts b/packages/kbn-test/src/kbn/index.ts index cf0cac046a8f..e562ab599c72 100644 --- a/packages/kbn-test/src/kbn/index.ts +++ b/packages/kbn-test/src/kbn/index.ts @@ -11,4 +11,5 @@ export { kibanaServerTestUser, adminTestUser, systemIndicesSuperuser, + kibanaServiceAccount, } from './users'; diff --git a/packages/kbn-test/src/kbn/users.ts b/packages/kbn-test/src/kbn/users.ts index b0db9e88ffc4..81adda598356 100644 --- a/packages/kbn-test/src/kbn/users.ts +++ b/packages/kbn-test/src/kbn/users.ts @@ -32,3 +32,13 @@ export const systemIndicesSuperuser = { username: SYSTEM_INDICES_SUPERUSER, password: env.TEST_ES_PASS || 'changeme', }; + +/** + * `kibana-dev` service account token for connecting to ESS + * See packages/kbn-es/src/ess_resources/README.md + */ +export const kibanaServiceAccount = { + token: + env.TEST_KIBANA_SERVICE_ACCOUNT_TOKEN || + 'AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA', +}; diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 7d2617900416..9131c3136379 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -9,7 +9,7 @@ import { resolve } from 'path'; import { format as formatUrl } from 'url'; import { REPO_ROOT } from '@kbn/repo-info'; -import { esTestConfig, kbnTestConfig, kibanaServerTestUser } from '@kbn/test'; +import { esTestConfig, kbnTestConfig, kibanaServiceAccount } from '@kbn/test'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; export default async () => { @@ -43,8 +43,7 @@ export default async () => { Object.entries(servers.elasticsearch).filter(([key]) => key.toLowerCase() !== 'auth') ) )}`, - `--elasticsearch.username=${kibanaServerTestUser.username}`, - `--elasticsearch.password=${kibanaServerTestUser.password}`, + `--elasticsearch.serviceAccountToken=${kibanaServiceAccount.token}`, '--telemetry.sendUsageTo=staging', `--logging.appenders.deprecation=${JSON.stringify({ type: 'console', From 1fa9515a1d4c17a0c7dcdfc522832b5117589687 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 21 Aug 2023 12:13:49 -0600 Subject: [PATCH 076/182] default cluster to ssl. disable http ssl --- packages/kbn-test/src/es/test_es_cluster.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 2f9489a1cfe2..a687ef161657 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -225,11 +225,11 @@ export function createTestEsCluster< } else if (esFrom === 'serverless') { return await firstNode.runServerless({ basePath, - esArgs: customEsArgs, - password, + esArgs: ['xpack.security.http.ssl.enabled=false', ...customEsArgs], port, clean: true, teardown: true, + ssl: true, }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; From 44f1da16d332832b89ed72d918c0ebefcc5128d2 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 22 Aug 2023 20:44:42 +0100 Subject: [PATCH 077/182] chore(NA): support sideloaded users login on ess --- packages/kbn-es/index.ts | 2 +- packages/kbn-es/src/ess_resources/operator_users.yml | 4 ++-- packages/kbn-es/src/ess_resources/users | 1 + packages/kbn-es/src/ess_resources/users_roles | 1 + packages/kbn-es/src/paths.ts | 4 ++++ packages/kbn-es/src/utils/docker.ts | 8 +++++++- packages/kbn-es/src/utils/ess_file_realm.ts | 10 ++++++++++ packages/kbn-es/src/utils/index.ts | 1 + packages/kbn-test/index.ts | 1 + packages/kbn-test/src/kbn/index.ts | 1 + packages/kbn-test/src/kbn/users.ts | 7 ++++++- .../observability/cypress/support/commands.ts | 2 +- 12 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 packages/kbn-es/src/ess_resources/users create mode 100644 packages/kbn-es/src/ess_resources/users_roles create mode 100644 packages/kbn-es/src/utils/ess_file_realm.ts diff --git a/packages/kbn-es/index.ts b/packages/kbn-es/index.ts index aed2ab7af41c..e8d3b2a4cfa1 100644 --- a/packages/kbn-es/index.ts +++ b/packages/kbn-es/index.ts @@ -8,4 +8,4 @@ export { run } from './src/cli'; export { Cluster } from './src/cluster'; -export { SYSTEM_INDICES_SUPERUSER } from './src/utils'; +export { SYSTEM_INDICES_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER_PASSWORD } from './src/utils'; diff --git a/packages/kbn-es/src/ess_resources/operator_users.yml b/packages/kbn-es/src/ess_resources/operator_users.yml index 6e13581aa2da..3071a55ed352 100644 --- a/packages/kbn-es/src/ess_resources/operator_users.yml +++ b/packages/kbn-es/src/ess_resources/operator_users.yml @@ -1,9 +1,9 @@ operator: - - usernames: ["elastic-admin"] + - usernames: ["kibana_serverless"] realm_type: "file" auth_type: "realm" - usernames: [ "elastic/kibana" ] realm_type: "_service_account" auth_type: "token" token_source: "file" - token_names: [ "kibana-dev" ] \ No newline at end of file + token_names: [ "kibana-dev" ] diff --git a/packages/kbn-es/src/ess_resources/users b/packages/kbn-es/src/ess_resources/users new file mode 100644 index 000000000000..a3b05bff4032 --- /dev/null +++ b/packages/kbn-es/src/ess_resources/users @@ -0,0 +1 @@ +kibana_serverless_superuser:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW diff --git a/packages/kbn-es/src/ess_resources/users_roles b/packages/kbn-es/src/ess_resources/users_roles new file mode 100644 index 000000000000..45a6d488d369 --- /dev/null +++ b/packages/kbn-es/src/ess_resources/users_roles @@ -0,0 +1 @@ +superuser:kibana_serverless_superuser diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 91e1cd5b2a4c..a2dabfb2e0b0 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -26,3 +26,7 @@ export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore'); export const ESS_OPERATOR_USERS_PATH = resolve(__dirname, './ess_resources/operator_users.yml'); export const ESS_SERVICE_TOKENS_PATH = resolve(__dirname, './ess_resources/service_tokens'); + +export const ESS_USERS_PATH = resolve(__dirname, './ess_resources/users'); + +export const ESS_USERS_ROLES_PATH = resolve(__dirname, './ess_resources/users_roles'); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 375b0ab5bb72..7cee62a98d0d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -17,7 +17,7 @@ import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_OPERATOR_USERS_PATH, ESS_SERVICE_TOKENS_PATH } from '../paths'; +import { ESS_USERS_PATH, ESS_USERS_ROLES_PATH, ESS_OPERATOR_USERS_PATH, ESS_SERVICE_TOKENS_PATH } from '../paths'; interface BaseOptions { tag?: string; @@ -386,6 +386,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles '--volume', `${ESS_OPERATOR_USERS_PATH}:${ESS_CONFIG_PATH}operator_users.yml`, + '--volume', + `${ESS_USERS_PATH}:${ESS_CONFIG_PATH}users`, + + '--volume', + `${ESS_USERS_ROLES_PATH}:${ESS_CONFIG_PATH}users_roles`, + '--volume', `${ESS_SERVICE_TOKENS_PATH}:${ESS_CONFIG_PATH}service_tokens`, ] diff --git a/packages/kbn-es/src/utils/ess_file_realm.ts b/packages/kbn-es/src/utils/ess_file_realm.ts new file mode 100644 index 000000000000..cb7c019240e2 --- /dev/null +++ b/packages/kbn-es/src/utils/ess_file_realm.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const KIBANA_SERVERLESS_SUPERUSER = 'kibana_serverless_superuser'; +export const KIBANA_SERVERLESS_SUPERUSER_PASSWORD = 'changeme'; diff --git a/packages/kbn-es/src/utils/index.ts b/packages/kbn-es/src/utils/index.ts index 25591a786603..cc46d0bf2827 100644 --- a/packages/kbn-es/src/utils/index.ts +++ b/packages/kbn-es/src/utils/index.ts @@ -17,3 +17,4 @@ export { buildSnapshot } from './build_snapshot'; export { archiveForPlatform } from './build_snapshot'; export * from './parse_timeout_to_ms'; export * from './docker'; +export * from './ess_file_realm'; diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index cae824dc92a6..3e99d9def116 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -39,6 +39,7 @@ export { adminTestUser, systemIndicesSuperuser, kibanaServiceAccount, + kibanaEssTestSuperUser, } from './src/kbn'; // @internal diff --git a/packages/kbn-test/src/kbn/index.ts b/packages/kbn-test/src/kbn/index.ts index e562ab599c72..42ede01151e6 100644 --- a/packages/kbn-test/src/kbn/index.ts +++ b/packages/kbn-test/src/kbn/index.ts @@ -12,4 +12,5 @@ export { adminTestUser, systemIndicesSuperuser, kibanaServiceAccount, + kibanaEssTestSuperUser, } from './users'; diff --git a/packages/kbn-test/src/kbn/users.ts b/packages/kbn-test/src/kbn/users.ts index 81adda598356..79b3881e890f 100644 --- a/packages/kbn-test/src/kbn/users.ts +++ b/packages/kbn-test/src/kbn/users.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { SYSTEM_INDICES_SUPERUSER } from '@kbn/es'; +import { SYSTEM_INDICES_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER_PASSWORD } from '@kbn/es'; const env = process.env; @@ -42,3 +42,8 @@ export const kibanaServiceAccount = { env.TEST_KIBANA_SERVICE_ACCOUNT_TOKEN || 'AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA', }; + +export const kibanaEssTestSuperUser = { + username: KIBANA_SERVERLESS_SUPERUSER, + password: KIBANA_SERVERLESS_SUPERUSER_PASSWORD, +}; diff --git a/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts b/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts index f9cea3121517..369d3bf1cc35 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts @@ -10,7 +10,7 @@ import 'cypress-axe'; Cypress.Commands.add('loginAsElasticUser', (path?: string) => { cy.visit(path ?? '/', { auth: { - username: 'elastic', + username: 'kibana_serverless_superuser', password: 'changeme', }, }); From 8bb285b45ddc1bc9bf8d96f79e6344de69806bdc Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 14:14:20 -0600 Subject: [PATCH 078/182] cleanup --- packages/kbn-es/src/paths.ts | 2 -- packages/kbn-es/src/utils/docker.ts | 7 ++++++- packages/kbn-test/index.ts | 2 +- packages/kbn-test/src/kbn/index.ts | 2 +- packages/kbn-test/src/kbn/users.ts | 8 ++++++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index a2dabfb2e0b0..29598673dae5 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -26,7 +26,5 @@ export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore'); export const ESS_OPERATOR_USERS_PATH = resolve(__dirname, './ess_resources/operator_users.yml'); export const ESS_SERVICE_TOKENS_PATH = resolve(__dirname, './ess_resources/service_tokens'); - export const ESS_USERS_PATH = resolve(__dirname, './ess_resources/users'); - export const ESS_USERS_ROLES_PATH = resolve(__dirname, './ess_resources/users_roles'); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 7cee62a98d0d..48bef86505a5 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -17,7 +17,12 @@ import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_USERS_PATH, ESS_USERS_ROLES_PATH, ESS_OPERATOR_USERS_PATH, ESS_SERVICE_TOKENS_PATH } from '../paths'; +import { + ESS_USERS_PATH, + ESS_USERS_ROLES_PATH, + ESS_OPERATOR_USERS_PATH, + ESS_SERVICE_TOKENS_PATH, +} from '../paths'; interface BaseOptions { tag?: string; diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index 3e99d9def116..503a7b3b8034 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -39,7 +39,7 @@ export { adminTestUser, systemIndicesSuperuser, kibanaServiceAccount, - kibanaEssTestSuperUser, + kibanaServerlessSuperuser, } from './src/kbn'; // @internal diff --git a/packages/kbn-test/src/kbn/index.ts b/packages/kbn-test/src/kbn/index.ts index 42ede01151e6..414f7d63554c 100644 --- a/packages/kbn-test/src/kbn/index.ts +++ b/packages/kbn-test/src/kbn/index.ts @@ -12,5 +12,5 @@ export { adminTestUser, systemIndicesSuperuser, kibanaServiceAccount, - kibanaEssTestSuperUser, + kibanaServerlessSuperuser, } from './users'; diff --git a/packages/kbn-test/src/kbn/users.ts b/packages/kbn-test/src/kbn/users.ts index 79b3881e890f..e1d7cef0f40d 100644 --- a/packages/kbn-test/src/kbn/users.ts +++ b/packages/kbn-test/src/kbn/users.ts @@ -6,7 +6,11 @@ * Side Public License, v 1. */ -import { SYSTEM_INDICES_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER_PASSWORD } from '@kbn/es'; +import { + SYSTEM_INDICES_SUPERUSER, + KIBANA_SERVERLESS_SUPERUSER, + KIBANA_SERVERLESS_SUPERUSER_PASSWORD, +} from '@kbn/es'; const env = process.env; @@ -43,7 +47,7 @@ export const kibanaServiceAccount = { 'AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA', }; -export const kibanaEssTestSuperUser = { +export const kibanaServerlessSuperuser = { username: KIBANA_SERVERLESS_SUPERUSER, password: KIBANA_SERVERLESS_SUPERUSER_PASSWORD, }; From ba4a7b2768aab66487d7d1564bb33c3aed0e4684 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 14:51:21 -0600 Subject: [PATCH 079/182] Update ess resources readme --- packages/kbn-es/src/ess_resources/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/ess_resources/README.md b/packages/kbn-es/src/ess_resources/README.md index 70fa7a232f4f..49118c5da7bb 100644 --- a/packages/kbn-es/src/ess_resources/README.md +++ b/packages/kbn-es/src/ess_resources/README.md @@ -1,4 +1,18 @@ -This was originally from the [ESS repository](https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/resources/README.service_tokens.md). +# Elasticsearch Serverless Resources +The resources in this directory are used for seeding Elasticsearch Serverless (ESS) images with users, roles and tokens for SSL and authentication. ESS requires file realm authentication, so we will bind mount them into the containers at `/usr/share/elasticsearch/config/`. + +## Default User + +The default superuser authentication to login to Kibana is: + +``` +username: kibana_serverless_superuser +password: changeme +``` + +## Service Account and Tokens + +This section for Service Accounts was originally from the [ESS repository](https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/resources/README.service_tokens.md). The "service_tokens" file contains this line: ``` From d7ebab79e89bf17b617b9422fd9a6c10dddffed8 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 15:10:59 -0600 Subject: [PATCH 080/182] Add roles and role mapping. Improve volume setup --- packages/kbn-es/src/paths.ts | 13 ++++++++++++ packages/kbn-es/src/utils/docker.ts | 33 +++++++++++------------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 29598673dae5..26cee48b11b3 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -26,5 +26,18 @@ export const ES_KEYSTORE_BIN = maybeUseBat('./bin/elasticsearch-keystore'); export const ESS_OPERATOR_USERS_PATH = resolve(__dirname, './ess_resources/operator_users.yml'); export const ESS_SERVICE_TOKENS_PATH = resolve(__dirname, './ess_resources/service_tokens'); + export const ESS_USERS_PATH = resolve(__dirname, './ess_resources/users'); export const ESS_USERS_ROLES_PATH = resolve(__dirname, './ess_resources/users_roles'); + +export const ESS_ROLES_PATH = resolve(__dirname, './ess_resources/roles.yml'); +export const ESS_ROLE_MAPPING_PATH = resolve(__dirname, './ess_resources/role_mapping.yml'); + +export const ESS_RESOURCES_PATHS = [ + ESS_OPERATOR_USERS_PATH, + ESS_ROLE_MAPPING_PATH, + ESS_ROLES_PATH, + ESS_SERVICE_TOKENS_PATH, + ESS_USERS_PATH, + ESS_USERS_ROLES_PATH, +]; diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 48bef86505a5..20d2be02fcff 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -17,12 +17,7 @@ import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { - ESS_USERS_PATH, - ESS_USERS_ROLES_PATH, - ESS_OPERATOR_USERS_PATH, - ESS_SERVICE_TOKENS_PATH, -} from '../paths'; +import { ESS_RESOURCES_PATHS } from '../paths'; interface BaseOptions { tag?: string; @@ -383,25 +378,21 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - return ['--volume', `${basePath}:/objectstore:z`].concat( - ssl - ? [ - ...getESp12Volume(), + const baseCmd = ['--volume', `${basePath}:/objectstore:z`]; - '--volume', - `${ESS_OPERATOR_USERS_PATH}:${ESS_CONFIG_PATH}operator_users.yml`, + if (ssl) { + const essResources = ESS_RESOURCES_PATHS.reduce((acc, path) => { + const fileName = path.split('/').at(-1); - '--volume', - `${ESS_USERS_PATH}:${ESS_CONFIG_PATH}users`, + acc.push('--volume', `${path}:${ESS_CONFIG_PATH}${fileName}`); - '--volume', - `${ESS_USERS_ROLES_PATH}:${ESS_CONFIG_PATH}users_roles`, + return acc; + }, []); - '--volume', - `${ESS_SERVICE_TOKENS_PATH}:${ESS_CONFIG_PATH}service_tokens`, - ] - : [] - ); + return baseCmd.concat(getESp12Volume(), essResources); + } + + return baseCmd; } /** From 6b80d9212bd914419396d73c2bcc48be76db12c0 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 15:13:11 -0600 Subject: [PATCH 081/182] Fix tests --- packages/kbn-es/src/utils/docker.test.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 503c5ac78284..b0604e3b8089 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -29,7 +29,7 @@ import { } from './docker'; import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log'; import { ES_P12_PATH } from '@kbn/dev-utils'; -import { ESS_OPERATOR_USERS_PATH, ESS_SERVICE_TOKENS_PATH } from '../paths'; +import { ESS_RESOURCES_PATHS } from '../paths'; jest.mock('execa'); const execa = jest.requireMock('execa'); @@ -378,14 +378,11 @@ describe('setupServerlessVolumes()', () => { const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath, ssl: true }); - expect(volumeCmd).toHaveLength(8); + expect(volumeCmd).toHaveLength(16); expect( - [ - `${baseEsPath}:/objectstore:z`, - ES_P12_PATH, - ESS_OPERATOR_USERS_PATH, - ESS_SERVICE_TOKENS_PATH, - ].every((path) => volumeCmd.some((cmd) => cmd.includes(path))) + [`${baseEsPath}:/objectstore:z`, ES_P12_PATH, ...ESS_RESOURCES_PATHS].every((path) => + volumeCmd.some((cmd) => cmd.includes(path)) + ) ).toBe(true); }); }); From 173059f274ae7776df92cf293ace24be007d451b Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 15:54:46 -0600 Subject: [PATCH 082/182] Add missing roles files --- packages/kbn-es/src/ess_resources/role_mapping.yml | 14 ++++++++++++++ packages/kbn-es/src/ess_resources/roles.yml | 0 2 files changed, 14 insertions(+) create mode 100644 packages/kbn-es/src/ess_resources/role_mapping.yml create mode 100644 packages/kbn-es/src/ess_resources/roles.yml diff --git a/packages/kbn-es/src/ess_resources/role_mapping.yml b/packages/kbn-es/src/ess_resources/role_mapping.yml new file mode 100644 index 000000000000..882bf8a76fd1 --- /dev/null +++ b/packages/kbn-es/src/ess_resources/role_mapping.yml @@ -0,0 +1,14 @@ +# Role mapping configuration file which has elasticsearch roles as keys +# that map to one or more user or group distinguished names + +#roleA: this is an elasticsearch role +# - groupA-DN this is a group distinguished name +# - groupB-DN +# - user1-DN this is the full user distinguished name + +#power_user: +# - "cn=admins,dc=example,dc=com" +#user: +# - "cn=users,dc=example,dc=com" +# - "cn=admins,dc=example,dc=com" +# - "cn=John Doe,cn=other users,dc=example,dc=com" \ No newline at end of file diff --git a/packages/kbn-es/src/ess_resources/roles.yml b/packages/kbn-es/src/ess_resources/roles.yml new file mode 100644 index 000000000000..e69de29bb2d1 From b7e86ca5b3378d5dda2b0c4df4b5c51206428112 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 16:36:25 -0600 Subject: [PATCH 083/182] Allow passing user into getUrlParts --- packages/kbn-test/kbn_test_config.ts | 11 +++++--- x-pack/test_serverless/shared/config.base.ts | 27 +++++++++++++------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/kbn-test/kbn_test_config.ts b/packages/kbn-test/kbn_test_config.ts index 1e656f334790..7a4d868d324a 100644 --- a/packages/kbn-test/kbn_test_config.ts +++ b/packages/kbn-test/kbn_test_config.ts @@ -18,12 +18,17 @@ export interface UrlParts { password?: string; } +interface UserAuth { + username: string; + password: string; +} + export const kbnTestConfig = new (class KbnTestConfig { getPort() { return this.getUrlParts().port; } - getUrlParts(): UrlParts { + getUrlParts(user: UserAuth = kibanaTestUser): UrlParts { // allow setting one complete TEST_KIBANA_URL for ES like https://elastic:changeme@example.com:9200 if (process.env.TEST_KIBANA_URL) { const testKibanaUrl = url.parse(process.env.TEST_KIBANA_URL); @@ -37,8 +42,8 @@ export const kbnTestConfig = new (class KbnTestConfig { }; } - const username = process.env.TEST_KIBANA_USERNAME || kibanaTestUser.username; - const password = process.env.TEST_KIBANA_PASSWORD || kibanaTestUser.password; + const username = process.env.TEST_KIBANA_USERNAME || user.username; + const password = process.env.TEST_KIBANA_PASSWORD || user.password; return { protocol: process.env.TEST_KIBANA_PROTOCOL || 'http', hostname: process.env.TEST_KIBANA_HOSTNAME || 'localhost', diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 6729fe4cbb86..f65ac79e10b4 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -9,12 +9,17 @@ import { resolve } from 'path'; import { format as formatUrl } from 'url'; import { REPO_ROOT } from '@kbn/repo-info'; -import { esTestConfig, kbnTestConfig, kibanaServiceAccount } from '@kbn/test'; +import { + esTestConfig, + kbnTestConfig, + kibanaServiceAccount, + kibanaServerlessSuperuser, +} from '@kbn/test'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; export default async () => { const servers = { - kibana: kbnTestConfig.getUrlParts(), + kibana: kbnTestConfig.getUrlParts(kibanaServerlessSuperuser), elasticsearch: esTestConfig.getUrlParts(), }; @@ -51,13 +56,17 @@ export default async () => { type: 'json', }, })}`, - `--logging.loggers=${JSON.stringify([ - { - name: 'elasticsearch.deprecation', - level: 'all', - appenders: ['deprecation'], - }, - ])}`, + /** + * ESS emits deprecation warnings for ssl.keystore.password. + * Need to mount a secure keystore into the images because ES_NOPASSWORD_P12_PATH doesn't work. + */ + // `--logging.loggers=${JSON.stringify([ + // { + // name: 'elasticsearch.deprecation', + // level: 'all', + // appenders: ['deprecation'], + // }, + // ])}`, '--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', `--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`, ], From 2fa1e587a48eacb320cf1402252b4e04410a43f7 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 16:48:00 -0600 Subject: [PATCH 084/182] Teardown cluster if non-default image --- packages/kbn-es/src/cluster.js | 2 +- packages/kbn-es/src/utils/docker.test.ts | 6 ++++-- packages/kbn-es/src/utils/docker.ts | 5 ++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index 54ad0595b8c7..dbbe3930c734 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -590,7 +590,7 @@ exports.Cluster = class Cluster { * Ideally would be async and an event like beforeExit or SIGINT, * but those events are not being triggered in FTR child process. */ - process.on('exit', () => teardownServerlessClusterSync(this._log)); + process.on('exit', () => teardownServerlessClusterSync(this._log, options)); } } diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index b0604e3b8089..fe6cfcf8159d 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -446,13 +446,15 @@ describe('stopServerlessCluster()', () => { }); describe('teardownServerlessClusterSync()', () => { + const defaultOptions = { basePath: 'foo/bar' }; + test('should kill running serverless nodes', () => { const nodes = ['es01', 'es02', 'es03']; execa.commandSync.mockImplementation(() => ({ stdout: nodes.join('\n'), })); - teardownServerlessClusterSync(log); + teardownServerlessClusterSync(log, defaultOptions); expect(execa.commandSync.mock.calls).toHaveLength(2); expect(execa.commandSync.mock.calls[0][0]).toEqual(expect.stringContaining(SERVERLESS_IMG)); @@ -464,7 +466,7 @@ describe('teardownServerlessClusterSync()', () => { stdout: '\n', })); - teardownServerlessClusterSync(log); + teardownServerlessClusterSync(log, defaultOptions); expect(execa.commandSync.mock.calls).toHaveLength(1); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 20d2be02fcff..91e74e7f05e9 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -478,10 +478,9 @@ export async function stopServerlessCluster(log: ToolingLog, nodes: string[]) { /** * Kill any serverless ES nodes which are running. */ -export function teardownServerlessClusterSync(log: ToolingLog) { - // TODO: this should use the resolved image +export function teardownServerlessClusterSync(log: ToolingLog, options: ServerlessOptions) { const { stdout } = execa.commandSync( - `docker ps --filter status=running --filter ancestor=${SERVERLESS_IMG} --quiet` + `docker ps --filter status=running --filter ancestor=${getServerlessImage(options)} --quiet` ); // Filter empty strings const runningNodes = stdout.split(/\r?\n/).filter((s) => s); From f149d0710facd4e9a862fa0b5ad379bf9be66008 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 22 Aug 2023 23:09:26 +0000 Subject: [PATCH 085/182] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- packages/kbn-es/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/index.ts b/packages/kbn-es/index.ts index e8d3b2a4cfa1..025bf3b5f013 100644 --- a/packages/kbn-es/index.ts +++ b/packages/kbn-es/index.ts @@ -8,4 +8,8 @@ export { run } from './src/cli'; export { Cluster } from './src/cluster'; -export { SYSTEM_INDICES_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER_PASSWORD } from './src/utils'; +export { + SYSTEM_INDICES_SUPERUSER, + KIBANA_SERVERLESS_SUPERUSER, + KIBANA_SERVERLESS_SUPERUSER_PASSWORD, +} from './src/utils'; From 98fa37cad00b40a017a7533b11a283af97ee1daa Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 17:39:39 -0600 Subject: [PATCH 086/182] Add CLI option to kill existing nodes on startup --- packages/kbn-es/src/cli_commands/docker.ts | 3 +- .../kbn-es/src/cli_commands/serverless.ts | 7 +-- packages/kbn-es/src/utils/docker.test.ts | 43 ++++++++++++++++++- packages/kbn-es/src/utils/docker.ts | 41 ++++++++++++++++-- 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/packages/kbn-es/src/cli_commands/docker.ts b/packages/kbn-es/src/cli_commands/docker.ts index 963d4278349b..7471b97f98d1 100644 --- a/packages/kbn-es/src/cli_commands/docker.ts +++ b/packages/kbn-es/src/cli_commands/docker.ts @@ -29,6 +29,7 @@ export const docker: Command = { --password Sets password for elastic user [default: ${password}] --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] --ssl Sets up SSL on Elasticsearch + --kill Kill running ES nodes if detected -E Additional key=value settings to pass to Elasticsearch -D Override Docker command @@ -55,7 +56,7 @@ export const docker: Command = { }, string: ['tag', 'image', 'D'], - boolean: ['ssl'], + boolean: ['ssl', 'kill'], default: defaults, }); diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index 0475910d3c74..6c7f5e9372bf 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -22,11 +22,12 @@ export const serverless: Command = { return dedent` Options: - --tag Image tag of ES Serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}] - --image Full path of ES Serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}] + --tag Image tag of ESS to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}] + --image Full path of ESS image to run, has precedence over tag. [default: ${SERVERLESS_IMG}] --clean Remove existing file system object store before running --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] --ssl Sets up SSL on Elasticsearch + --kill Kill running ESS nodes if detected -E Additional key=value settings to pass to Elasticsearch Examples: @@ -51,7 +52,7 @@ export const serverless: Command = { }, string: ['tag', 'image'], - boolean: ['clean', 'ssl'], + boolean: ['clean', 'ssl', 'kill'], default: defaults, }); diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index fe6cfcf8159d..2fbba684d3d7 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -12,6 +12,7 @@ import { stat } from 'fs/promises'; import { DOCKER_IMG, + detectRunningNodes, maybeCreateDockerNetwork, maybePullDockerImage, resolveDockerCmd, @@ -233,6 +234,44 @@ describe('maybePullDockerImage()', () => { }); }); +describe('detectRunningNodes()', () => { + const nodes = ['es01', 'es02', 'es03']; + + test('should not error if no nodes detected', async () => { + execa.mockImplementationOnce(() => Promise.resolve({ stdout: '' })); + + await detectRunningNodes(log, {}); + + expect(execa.mock.calls).toHaveLength(1); + expect(execa.mock.calls[0][1]).toEqual(expect.arrayContaining(['ps', '--quiet', '--filter'])); + }); + + test('should kill nodes if detected and kill passed', async () => { + execa.mockImplementationOnce(() => + Promise.resolve({ + stdout: nodes.join('\n'), + }) + ); + + await detectRunningNodes(log, { kill: true }); + + expect(execa.mock.calls).toHaveLength(2); + expect(execa.mock.calls[1][1]).toEqual(expect.arrayContaining(nodes.concat('kill'))); + }); + + test('should error if nodes detected and kill not passed', async () => { + execa.mockImplementationOnce(() => + Promise.resolve({ + stdout: nodes.join('\n'), + }) + ); + + await expect(detectRunningNodes(log, {})).rejects.toThrowErrorMatchingInlineSnapshot( + `"ES has already been started"` + ); + }); +}); + describe('resolveEsArgs()', () => { const defaultEsArgs: Array<[string, string]> = [ ['foo', 'bar'], @@ -427,7 +466,7 @@ describe('runServerlessCluster()', () => { await runServerlessCluster(log, { basePath: baseEsPath }); // setupDocker execa calls then run three nodes - expect(execa.mock.calls).toHaveLength(6); + expect(execa.mock.calls).toHaveLength(7); }); }); @@ -499,6 +538,6 @@ describe('runDockerContainer()', () => { await expect(runDockerContainer(log, {})).resolves.toEqual({ stdout: '' }); // setupDocker execa calls then run container - expect(execa.mock.calls).toHaveLength(4); + expect(execa.mock.calls).toHaveLength(5); }); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 91e74e7f05e9..cdbbfb07d68f 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -24,6 +24,7 @@ interface BaseOptions { image?: string; port?: number; ssl?: boolean; + kill?: boolean; } export interface DockerOptions extends EsClusterExecOptions, BaseOptions { @@ -303,11 +304,45 @@ export async function maybePullDockerImage(log: ToolingLog, image: string) { }); } +export async function detectRunningNodes( + log: ToolingLog, + options: ServerlessOptions | DockerOptions +) { + const namesCmd = SERVERLESS_NODES.reduce((acc, { name }) => { + acc.push('--filter', `name=${name}`); + + return acc; + }, []); + + const { stdout } = await execa('docker', ['ps', '--quiet'].concat(namesCmd)); + const runningNodes = stdout.split(/\r?\n/).filter((s) => s); + + if (runningNodes.length) { + if (options.kill) { + log.info(chalk.bold('Running ES Nodes detected, killing.')); + await execa('docker', ['kill'].concat(runningNodes)); + + return; + } + + throw createCliError('ES has already been started'); + } +} + /** * Common setup for Docker and Serverless containers */ -async function setupDocker(log: ToolingLog, image: string) { +async function setupDocker({ + log, + image, + options, +}: { + log: ToolingLog; + image: string; + options: ServerlessOptions | DockerOptions; +}) { await verifyDockerInstalled(log); + await detectRunningNodes(log, options); await maybeCreateDockerNetwork(log); await maybePullDockerImage(log, image); } @@ -440,7 +475,7 @@ export async function runServerlessEsNode( */ export async function runServerlessCluster(log: ToolingLog, options: ServerlessOptions) { const image = getServerlessImage(options); - await setupDocker(log, image); + await setupDocker({ log, image, options }); const volumeCmd = await setupServerlessVolumes(log, options); @@ -523,7 +558,7 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions if (!options.dockerCmd) { image = getDockerImage(options); - await setupDocker(log, image); + await setupDocker({ log, image, options }); } const dockerCmd = resolveDockerCmd(options, image); From e6dcac39b7e6e52c124aad758020e9d860773536 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 22 Aug 2023 17:59:56 -0600 Subject: [PATCH 087/182] Automatically attach logs --- packages/kbn-es/src/cli_commands/serverless.ts | 3 ++- packages/kbn-es/src/utils/docker.test.ts | 6 +++--- packages/kbn-es/src/utils/docker.ts | 15 +++++++++++++-- packages/kbn-test/src/es/test_es_cluster.ts | 1 + 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index 6c7f5e9372bf..2e0651977aea 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -28,6 +28,7 @@ export const serverless: Command = { --port The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}] --ssl Sets up SSL on Elasticsearch --kill Kill running ESS nodes if detected + --background Start ESS without attaching to the first node's logs -E Additional key=value settings to pass to Elasticsearch Examples: @@ -52,7 +53,7 @@ export const serverless: Command = { }, string: ['tag', 'image'], - boolean: ['clean', 'ssl', 'kill'], + boolean: ['clean', 'ssl', 'kill', 'background'], default: defaults, }); diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 2fbba684d3d7..af881fbef99d 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -267,7 +267,7 @@ describe('detectRunningNodes()', () => { ); await expect(detectRunningNodes(log, {})).rejects.toThrowErrorMatchingInlineSnapshot( - `"ES has already been started"` + `"ES has already been started, pass --kill to automatically stop the nodes on startup."` ); }); }); @@ -465,8 +465,8 @@ describe('runServerlessCluster()', () => { await runServerlessCluster(log, { basePath: baseEsPath }); - // setupDocker execa calls then run three nodes - expect(execa.mock.calls).toHaveLength(7); + // setupDocker execa calls then run three nodes and attach logger + expect(execa.mock.calls).toHaveLength(8); }); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index cdbbfb07d68f..588ab7ec9bd4 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -35,6 +35,7 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions { clean?: boolean; basePath: string; teardown?: boolean; + background?: boolean; } interface ServerlessEsNodeArgs { @@ -319,13 +320,15 @@ export async function detectRunningNodes( if (runningNodes.length) { if (options.kill) { - log.info(chalk.bold('Running ES Nodes detected, killing.')); + log.info(chalk.bold('Killing running ES Nodes.')); await execa('docker', ['kill'].concat(runningNodes)); return; } - throw createCliError('ES has already been started'); + throw createCliError( + 'ES has already been started, pass --kill to automatically stop the nodes on startup.' + ); } } @@ -498,6 +501,14 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO Stop the cluster: ${chalk.bold(`docker container stop ${nodeNames.join(' ')}`)} `); + if (!options.background) { + // The ESS cluster has to be started detached, so we attach a logger afterwards for output + await execa('docker', ['logs', '-f', SERVERLESS_NODES[0].name], { + // inherit is required to show Docker output and Java console output for pw, enrollment token, etc + stdio: ['ignore', 'inherit', 'inherit'], + }); + } + return nodeNames; } diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index a687ef161657..8adb22b2724d 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -230,6 +230,7 @@ export function createTestEsCluster< clean: true, teardown: true, ssl: true, + background: true, }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; From 13399f14fec07dd27a90dcf7e27feea19a7862a3 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 04:04:46 +0100 Subject: [PATCH 088/182] chore(NA): adding system_indices_superuser --- packages/kbn-es/src/ess_resources/users | 1 + packages/kbn-es/src/ess_resources/users_roles | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/kbn-es/src/ess_resources/users b/packages/kbn-es/src/ess_resources/users index a3b05bff4032..d982cc51aec7 100644 --- a/packages/kbn-es/src/ess_resources/users +++ b/packages/kbn-es/src/ess_resources/users @@ -1 +1,2 @@ kibana_serverless_superuser:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW +system_indices_superuser:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW diff --git a/packages/kbn-es/src/ess_resources/users_roles b/packages/kbn-es/src/ess_resources/users_roles index 45a6d488d369..119278073c7d 100644 --- a/packages/kbn-es/src/ess_resources/users_roles +++ b/packages/kbn-es/src/ess_resources/users_roles @@ -1 +1,2 @@ superuser:kibana_serverless_superuser +superuser:system_indices_superuser From c78abea00d3fecc67f7f19325d4b727d73b300c9 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 04:23:59 +0100 Subject: [PATCH 089/182] chore(NA): whitelist users on operators --- packages/kbn-es/src/ess_resources/operator_users.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-es/src/ess_resources/operator_users.yml b/packages/kbn-es/src/ess_resources/operator_users.yml index 3071a55ed352..63f6a6e99972 100644 --- a/packages/kbn-es/src/ess_resources/operator_users.yml +++ b/packages/kbn-es/src/ess_resources/operator_users.yml @@ -1,5 +1,5 @@ operator: - - usernames: ["kibana_serverless"] + - usernames: ["kibana_serverless_superuser", "system_indices_superuser"] realm_type: "file" auth_type: "realm" - usernames: [ "elastic/kibana" ] From 11477f53cfeb685a194317a31bfdcf68423b1c94 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 14:24:19 +0100 Subject: [PATCH 090/182] chore(NA): specific role for system_indices_superuser --- packages/kbn-es/src/ess_resources/roles.yml | 13 +++++++++++++ packages/kbn-es/src/ess_resources/users_roles | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/ess_resources/roles.yml b/packages/kbn-es/src/ess_resources/roles.yml index e69de29bb2d1..1890e2a741d1 100644 --- a/packages/kbn-es/src/ess_resources/roles.yml +++ b/packages/kbn-es/src/ess_resources/roles.yml @@ -0,0 +1,13 @@ +--- +system_indices_superuser: + refresh: 'wait_for' + cluster: ['all'] + indices: + - names: ['*'] + privileges: ['all'] + allow_restricted_indices: true + applications: + - application: '*' + privileges: ['*'] + resources: '*' + run_as: '*' diff --git a/packages/kbn-es/src/ess_resources/users_roles b/packages/kbn-es/src/ess_resources/users_roles index 119278073c7d..622b8958772b 100644 --- a/packages/kbn-es/src/ess_resources/users_roles +++ b/packages/kbn-es/src/ess_resources/users_roles @@ -1,2 +1,2 @@ superuser:kibana_serverless_superuser -superuser:system_indices_superuser +system_indices_superuser:system_indices_superuser From 2bdaf3a501dc04fdd12f3565df439dd7fbcd9460 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 15:26:18 +0100 Subject: [PATCH 091/182] chore(NA): update roles --- packages/kbn-es/src/ess_resources/roles.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/kbn-es/src/ess_resources/roles.yml b/packages/kbn-es/src/ess_resources/roles.yml index 1890e2a741d1..644cdf79d64b 100644 --- a/packages/kbn-es/src/ess_resources/roles.yml +++ b/packages/kbn-es/src/ess_resources/roles.yml @@ -1,6 +1,5 @@ --- system_indices_superuser: - refresh: 'wait_for' cluster: ['all'] indices: - names: ['*'] @@ -9,5 +8,5 @@ system_indices_superuser: applications: - application: '*' privileges: ['*'] - resources: '*' - run_as: '*' + resources: ['*'] + run_as: ['*'] From 657c47c382b8f432b53a70ff9a19a98201a47ed5 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 15:49:23 +0100 Subject: [PATCH 092/182] chore(NA): apply changes to parallel script from security solution --- .../security_solution/scripts/run_cypress/parallel.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 19f409fb6a56..b07c6f1b6f4f 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -331,7 +331,7 @@ ${JSON.stringify(config.getAll(), null, 2)} config, log, name: `ftr-${esPort}`, - esFrom: 'snapshot', + esFrom: config.get('esTestCluster')?.from || 'snapshot', onEarlyExit, }), { retries: 2, forever: false } @@ -469,11 +469,7 @@ ${JSON.stringify(cyCustomEnv, null, 2)} return result; }, { - concurrency: (argv.concurrency as number | undefined) - ? (argv.concurrency as number) - : !isOpen - ? 2 - : 1, + concurrency: 1, } ).then((results) => { renderSummaryTable(results as CypressCommandLine.CypressRunResult[]); From 8e1a934b3aebfe67bd6c96af070c4bb626b86fbb Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 17:42:45 +0100 Subject: [PATCH 093/182] chore(NA): login on docker --- .buildkite/scripts/steps/functional/common.sh | 4 ++++ .buildkite/scripts/steps/functional/security_serverless.sh | 3 --- .buildkite/scripts/steps/functional/serverless_ftr.sh | 3 --- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/steps/functional/common.sh b/.buildkite/scripts/steps/functional/common.sh index f9b77890030c..e6d13190b32c 100755 --- a/.buildkite/scripts/steps/functional/common.sh +++ b/.buildkite/scripts/steps/functional/common.sh @@ -21,3 +21,7 @@ if [[ -d "$cacheDir" ]]; then fi is_test_execution_step + +# logins into docker as a common step for functional tests +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT diff --git a/.buildkite/scripts/steps/functional/security_serverless.sh b/.buildkite/scripts/steps/functional/security_serverless.sh index 777338ac3e50..22b40c0f0c96 100644 --- a/.buildkite/scripts/steps/functional/security_serverless.sh +++ b/.buildkite/scripts/steps/functional/security_serverless.sh @@ -8,9 +8,6 @@ source .buildkite/scripts/steps/functional/common_cypress.sh export JOB=kibana-serverless-security-cypress export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} -echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co -trap 'docker logout docker.elastic.co' EXIT - echo "--- Security Serverless Cypress Tests" yarn --cwd x-pack/test/security_solution_cypress cypress:run:serverless diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh index 00515929a53e..ba75fe6034ca 100755 --- a/.buildkite/scripts/steps/functional/serverless_ftr.sh +++ b/.buildkite/scripts/steps/functional/serverless_ftr.sh @@ -6,9 +6,6 @@ source .buildkite/scripts/steps/functional/common.sh export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT" -echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co -trap 'docker logout docker.elastic.co' EXIT - if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then SERVERLESS_CONFIGS=( "x-pack/test_serverless/api_integration/test_suites/search/config.ts" From f864cd497d4baf9bf95e8a5946773d8d1bf079e9 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 23 Aug 2023 19:20:58 +0100 Subject: [PATCH 094/182] chore(NA): more predefined roles --- packages/kbn-es/src/ess_resources/roles.yml | 780 ++++++++++++++++++++ 1 file changed, 780 insertions(+) diff --git a/packages/kbn-es/src/ess_resources/roles.yml b/packages/kbn-es/src/ess_resources/roles.yml index 644cdf79d64b..84307eef3564 100644 --- a/packages/kbn-es/src/ess_resources/roles.yml +++ b/packages/kbn-es/src/ess_resources/roles.yml @@ -10,3 +10,783 @@ system_indices_superuser: privileges: ['*'] resources: ['*'] run_as: ['*'] + +# ----- +# Source: https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/roles.yml +# and: https://github.com/elastic/project-controller/blob/main/internal/project/esproject/config/roles.yml +# ----- +viewer: + cluster: [] + indices: + - names: + - "*" + privileges: + - read + - names: + - "/~(([.]|ilm-history-).*)/" + privileges: + - "read" + - "view_index_metadata" + allow_restricted_indices: false + - names: + - ".siem-signals*" + - ".lists-*" + - ".items-*" + privileges: + - "read" + - "view_index_metadata" + allow_restricted_indices: false + - names: + - ".alerts*" + - ".preview.alerts*" + privileges: + - "read" + - "view_index_metadata" + allow_restricted_indices: false + applications: + - application: "kibana-.kibana" + privileges: + - "read" + resources: + - "*" + run_as: [] +editor: + cluster: [] + indices: + - names: + - "/~(([.]|ilm-history-).*)/" + privileges: + - "read" + - "view_index_metadata" + allow_restricted_indices: false + - names: + - "observability-annotations" + privileges: + - "read" + - "view_index_metadata" + - "write" + allow_restricted_indices: false + - names: + - ".siem-signals*" + - ".lists-*" + - ".items-*" + privileges: + - "read" + - "view_index_metadata" + - "write" + - "maintenance" + allow_restricted_indices: false + - names: + - ".internal.alerts*" + - ".alerts*" + - ".internal.preview.alerts*" + - ".preview.alerts*" + privileges: + - "read" + - "view_index_metadata" + - "write" + - "maintenance" + allow_restricted_indices: false + applications: + - application: "kibana-.kibana" + privileges: + - "all" + resources: + - "*" + run_as: [] + +# ----- +# Source: https://github.com/elastic/project-controller/blob/main/internal/project/security/config/roles.yml +# ----- +t1_analyst: + cluster: + indices: + - names: + - ".alerts-security*" + - ".siem-signals-*" + privileges: + - read + - write + - maintenance + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + - metrics-endpoint.metadata_current_* + - ".fleet-agents*" + - ".fleet-actions*" + privileges: + - read + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - read + - read_alerts + - endpoint_list_read + resources: "*" + - application: securitySolutionCases + privileges: + - read + resources: "*" + - application: actions + privileges: + - read + resources: "*" + - application: builtInAlerts + privileges: + - read + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + - application: osquery + privileges: + - read + - run_saved_queries + resources: "*" + +t2_analyst: + cluster: + indices: + - names: + - .alerts-security* + - .siem-signals-* + privileges: + - read + - write + - maintenance + - names: + - .lists* + - .items* + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - read + - read_alerts + - endpoint_list_read + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - read + resources: "*" + - application: builtInAlerts + privileges: + - read + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + - application: osquery + privileges: + - read + - run_saved_queries + resources: "*" + +t3_analyst: + cluster: + indices: + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + privileges: + - read + - write + - names: + - .alerts-security* + - .siem-signals-* + privileges: + - read + - write + - names: + - .lists* + - .items* + privileges: + - read + - write + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - crud_alerts + - endpoint_list_all + - trusted_applications_all + - event_filters_all + - host_isolation_exceptions_all + - blocklist_all + - policy_management_read # Elastic Defend Policy Management + - host_isolation_all + - process_operations_all + - actions_log_management_all # Response actions history + - file_operations_all + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - read + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: osquery + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + +threat_intelligence_analyst: + cluster: + indices: + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - .lists* + - .items* + - packetbeat-* + - winlogbeat-* + privileges: + - read + - names: + - .alerts-security* + - .siem-signals-* + privileges: + - read + - write + - maintenance + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - read + - read_alerts + - endpoint_list_read + - blocklist_all + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - read + resources: "*" + - application: builtInAlerts + privileges: + - read + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + - application: osquery + privileges: + - all + resources: "*" + +rule_author: + cluster: + indices: + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + privileges: + - read + - write + - names: + - .alerts-security* + - .siem-signals-* + - .internal.preview.alerts-security* + - .preview.alerts-security* + privileges: + - read + - write + - maintenance + - view_index_metadata + - names: + - .lists* + - .items* + privileges: + - read + - write + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - crud_alerts + - policy_management_all + - endpoint_list_all + - trusted_applications_all + - event_filters_all + - host_isolation_exceptions_read + - blocklist_all + - actions_log_management_read + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - read + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + +soc_manager: + cluster: + indices: + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + privileges: + - read + - write + - names: + - .alerts-security* + - .siem-signals-* + - .preview.alerts-security* + - .internal.preview.alerts-security* + privileges: + - read + - write + - manage + - names: + - .lists* + - .items* + privileges: + - read + - write + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - crud_alerts + - policy_management_all + - endpoint_list_all + - trusted_applications_all + - event_filters_all + - host_isolation_exceptions_all + - blocklist_all + - host_isolation_all + - process_operations_all + - actions_log_management_all + - file_operations_all + - execute_operations_all + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - all + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + - application: osquery + privileges: + - all + resources: "*" + - application: savedObjectsManagement + privileges: + - all + resources: "*" + +detections_admin: + cluster: + indices: + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + - .lists* + - .items* + - .alerts-security* + - .siem-signals-* + - .preview.alerts-security* + - .internal.preview.alerts-security* + privileges: + - read + - write + - manage + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + applications: + - application: ml + privileges: + - all + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - crud_alerts + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - read + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: dev_tools + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + +platform_engineer: + cluster: + - manage + indices: + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + - .lists* + - .items* + - .alerts-security* + - .siem-signals-* + - .preview.alerts-security* + - .internal.preview.alerts-security* + privileges: + - all + applications: + - application: ml + privileges: + - all + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - crud_alerts + - policy_management_all + - endpoint_list_all + - trusted_applications_all + - event_filters_all + - host_isolation_exceptions_all + - blocklist_all + - actions_log_management_read + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - all + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: fleet + privileges: + - all + resources: "*" + - application: fleetv2 + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + - application: osquery + privileges: + - all + resources: "*" + +endpoint_operations_analyst: + cluster: + indices: + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + - .lists* + - .items* + privileges: + - read + - names: + - .alerts-security* + - .siem-signals-* + - .preview.alerts-security* + - .internal.preview.alerts-security* + privileges: + - read + - write + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - policy_management_all + - endpoint_list_all + - trusted_applications_all + - event_filters_all + - host_isolation_exceptions_all + - blocklist_all + - host_isolation_all + - process_operations_all + - actions_log_management_all # Response History + - file_operations_all + - execute_operations_all # Execute + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - all + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: osquery + privileges: + - all + resources: "*" + - application: fleet + privileges: + - all + resources: "*" + - application: fleetv2 + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" + +endpoint_policy_manager: + cluster: + indices: + - names: + - metrics-endpoint.metadata_current_* + - .fleet-agents* + - .fleet-actions* + privileges: + - read + - names: + - apm-*-transaction* + - traces-apm* + - auditbeat-* + - endgame-* + - filebeat-* + - logs-* + - packetbeat-* + - winlogbeat-* + - .lists* + - .items* + privileges: + - read + - names: + - .alerts-security* + - .siem-signals-* + - .preview.alerts-security* + - .internal.preview.alerts-security* + privileges: + - read + - write + - manage + applications: + - application: ml + privileges: + - read + resources: "*" + - application: siem + privileges: + - all + - read_alerts + - crud_alerts + - policy_management_all + - trusted_applications_all + - event_filters_all + - host_isolation_exceptions_all + - blocklist_all + - endpoint_list_all + resources: "*" + - application: securitySolutionCases + privileges: + - all + resources: "*" + - application: actions + privileges: + - all + resources: "*" + - application: builtInAlerts + privileges: + - all + resources: "*" + - application: osquery + privileges: + - all + resources: "*" + - application: fleet + privileges: + - all + resources: "*" + - application: fleetv2 + privileges: + - all + resources: "*" + - application: spaces + privileges: + - all + resources: "*" From 281d994b79c65fe17deb285981d39fec6b54da85 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 24 Aug 2023 17:01:43 +0100 Subject: [PATCH 095/182] chore(NA): skip unsupported setting --- x-pack/test_serverless/shared/config.base.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 89b08cfd34ea..dee8dd567359 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -46,7 +46,8 @@ export default async () => { 'xpack.security.authc.realms.jwt.jwt1.order=-98', `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`, 'xpack.security.authc.realms.jwt.jwt1.client_authentication.type=shared_secret', - `xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret=my_super_secret`, + // We need a side loaded Elasticsearch keystore for this secret + // `xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret=my_super_secret`, `xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/`, `xpack.security.authc.realms.jwt.jwt1.allowed_subjects=elastic-agent`, 'xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch', From 37cf4758e66fea179ae3f1656ab530aa066f66e0 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 24 Aug 2023 10:35:40 -0600 Subject: [PATCH 096/182] volume test fix --- packages/kbn-es/src/utils/docker.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index af881fbef99d..4c23970e5ef5 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -417,12 +417,13 @@ describe('setupServerlessVolumes()', () => { const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath, ssl: true }); + const requiredPaths = [`${baseEsPath}:/objectstore:z`, ES_P12_PATH, ...ESS_RESOURCES_PATHS]; + const pathsNotIncludedInCmd = requiredPaths.filter( + (path) => !volumeCmd.some((cmd) => cmd.includes(path)) + ); + expect(volumeCmd).toHaveLength(16); - expect( - [`${baseEsPath}:/objectstore:z`, ES_P12_PATH, ...ESS_RESOURCES_PATHS].every((path) => - volumeCmd.some((cmd) => cmd.includes(path)) - ) - ).toBe(true); + expect(pathsNotIncludedInCmd).toEqual([]); }); }); From 033fbbd875dd7d5f89fa3bab2d9eaa43ea16c06a Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 24 Aug 2023 17:45:27 +0100 Subject: [PATCH 097/182] Revert "Allow Kibana to restrict the usage of JWT for a predefined set of routes only. (#163806)" This reverts commit 5aee5da843fc1e7747b66bb45e8ebf7a808c144d. --- .github/CODEOWNERS | 2 +- .../src/routes/status.ts | 5 +- .../server/routes/stats/stats.ts | 5 +- .../authentication/authenticator.test.ts | 27 +---- .../server/authentication/authenticator.ts | 8 +- .../authentication/providers/http.test.ts | 108 ------------------ .../server/authentication/providers/http.ts | 33 ------ x-pack/plugins/security/server/config.test.ts | 90 --------------- x-pack/plugins/security/server/config.ts | 5 - x-pack/plugins/security/server/routes/tags.ts | 6 - .../background_task_utilization.test.ts | 2 - .../routes/background_task_utilization.ts | 3 - .../server/routes/metrics.test.ts | 1 - .../task_manager/server/routes/metrics.ts | 3 - .../packages/helpers/kibana.jsonc | 2 +- .../test_suites/common/index.ts | 1 - .../common/security/authentication_http.ts | 51 --------- x-pack/test_serverless/shared/config.base.ts | 21 +--- 18 files changed, 10 insertions(+), 363 deletions(-) delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 911bfb5161dc..89571e200ea0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -595,7 +595,7 @@ packages/kbn-search-api-panels @elastic/enterprise-search-frontend examples/search_examples @elastic/kibana-data-discovery packages/kbn-search-response-warnings @elastic/kibana-data-discovery x-pack/plugins/searchprofiler @elastic/platform-deployment-management -x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security +x-pack/test/security_api_integration/packages/helpers @elastic/kibana-core x-pack/plugins/security @elastic/kibana-security x-pack/plugins/security_solution_ess @elastic/security-solution x-pack/test/cases_api_integration/common/plugins/security_solution @elastic/response-ops diff --git a/packages/core/status/core-status-server-internal/src/routes/status.ts b/packages/core/status/core-status-server-internal/src/routes/status.ts index e06d667b4c78..403686bdf268 100644 --- a/packages/core/status/core-status-server-internal/src/routes/status.ts +++ b/packages/core/status/core-status-server-internal/src/routes/status.ts @@ -82,10 +82,7 @@ export const registerStatusRoute = ({ path: '/api/status', options: { authRequired: 'optional', - // The `api` tag ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page. - // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to - // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. - tags: ['api', 'security:acceptJWT'], + tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page access: 'public', // needs to be public to allow access from "system" users like k8s readiness probes. }, validate: { diff --git a/src/plugins/usage_collection/server/routes/stats/stats.ts b/src/plugins/usage_collection/server/routes/stats/stats.ts index 6e4a60621603..8c32003f3809 100644 --- a/src/plugins/usage_collection/server/routes/stats/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats/stats.ts @@ -55,10 +55,7 @@ export function registerStatsRoute({ path: '/api/stats', options: { authRequired: !config.allowAnonymous, - // The `api` tag ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page. - // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to - // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. - tags: ['api', 'security:acceptJWT'], + tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page access: 'public', // needs to be public to allow access from "system" users like metricbeat. }, validate: { diff --git a/x-pack/plugins/security/server/authentication/authenticator.test.ts b/x-pack/plugins/security/server/authentication/authenticator.test.ts index fbc31e588dc5..2de2fdbf4df2 100644 --- a/x-pack/plugins/security/server/authentication/authenticator.test.ts +++ b/x-pack/plugins/security/server/authentication/authenticator.test.ts @@ -62,14 +62,12 @@ function getMockOptions({ selector, accessAgreementMessage, customLogoutURL, - configContext = {}, }: { providers?: Record | string[]; http?: Partial; selector?: AuthenticatorOptions['config']['authc']['selector']; accessAgreementMessage?: string; customLogoutURL?: string; - configContext?: Record; } = {}) { const auditService = auditServiceMock.create(); auditLogger = auditLoggerMock.create(); @@ -88,10 +86,10 @@ function getMockOptions({ loggers: loggingSystemMock.create(), getServerBaseURL: jest.fn(), config: createConfig( - ConfigSchema.validate( - { authc: { selector, providers, http }, ...accessAgreementObj }, - configContext - ), + ConfigSchema.validate({ + authc: { selector, providers, http }, + ...accessAgreementObj, + }), loggingSystemMock.create().get(), { isTLSEnabled: false } ), @@ -319,23 +317,6 @@ describe('Authenticator', () => { }); }); - it('includes JWT options if specified', () => { - new Authenticator( - getMockOptions({ - providers: { basic: { basic1: { order: 0 } } }, - http: { jwt: { taggedRoutesOnly: true } }, - configContext: { serverless: true }, - }) - ); - - expect( - jest.requireMock('./providers/http').HTTPAuthenticationProvider - ).toHaveBeenCalledWith(expect.anything(), { - supportedSchemes: new Set(['apikey', 'bearer', 'basic']), - jwt: { taggedRoutesOnly: true }, - }); - }); - it('does not include additional schemes if `autoSchemesEnabled` is disabled', () => { new Authenticator( getMockOptions({ diff --git a/x-pack/plugins/security/server/authentication/authenticator.ts b/x-pack/plugins/security/server/authentication/authenticator.ts index 6fa811397a81..24329c0e7575 100644 --- a/x-pack/plugins/security/server/authentication/authenticator.ts +++ b/x-pack/plugins/security/server/authentication/authenticator.ts @@ -648,13 +648,7 @@ export class Authenticator { throw new Error(`Provider name "${options.name}" is reserved.`); } - this.providers.set( - options.name, - new HTTPAuthenticationProvider(options, { - supportedSchemes, - jwt: this.options.config.authc.http.jwt, - }) - ); + this.providers.set(options.name, new HTTPAuthenticationProvider(options, { supportedSchemes })); } /** diff --git a/x-pack/plugins/security/server/authentication/providers/http.test.ts b/x-pack/plugins/security/server/authentication/providers/http.test.ts index 90ff62294ff3..c1e7ba662c51 100644 --- a/x-pack/plugins/security/server/authentication/providers/http.test.ts +++ b/x-pack/plugins/security/server/authentication/providers/http.test.ts @@ -15,7 +15,6 @@ import { mockAuthenticationProviderOptions } from './base.mock'; import { HTTPAuthenticationProvider } from './http'; import { mockAuthenticatedUser } from '../../../common/model/authenticated_user.mock'; import { securityMock } from '../../mocks'; -import { ROUTE_TAG_ACCEPT_JWT } from '../../routes/tags'; import { AuthenticationResult } from '../authentication_result'; import { DeauthenticationResult } from '../deauthentication_result'; @@ -145,113 +144,6 @@ describe('HTTPAuthenticationProvider', () => { } }); - it('succeeds for JWT authentication if not restricted to tagged routes.', async () => { - const header = 'Bearer header.body.signature'; - const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } }); - const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } }); - - const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); - mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); - mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); - mockOptions.client.asScoped.mockClear(); - - const provider = new HTTPAuthenticationProvider(mockOptions, { - supportedSchemes: new Set(['bearer']), - }); - - await expect(provider.authenticate(request)).resolves.toEqual( - AuthenticationResult.succeeded({ - ...user, - authentication_provider: { type: 'http', name: 'http' }, - }) - ); - - expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); - - expect(request.headers.authorization).toBe(header); - }); - - it('succeeds for non-JWT authentication if JWT restricted to tagged routes.', async () => { - const header = 'Basic xxx'; - const user = mockAuthenticatedUser(); - const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } }); - - const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); - mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); - mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); - mockOptions.client.asScoped.mockClear(); - - const provider = new HTTPAuthenticationProvider(mockOptions, { - supportedSchemes: new Set(['bearer', 'basic']), - jwt: { taggedRoutesOnly: true }, - }); - - await expect(provider.authenticate(request)).resolves.toEqual( - AuthenticationResult.succeeded({ - ...user, - authentication_provider: { type: 'http', name: 'http' }, - }) - ); - - expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); - - expect(request.headers.authorization).toBe(header); - }); - - it('succeeds for JWT authentication if restricted to tagged routes and route is tagged.', async () => { - const header = 'Bearer header.body.signature'; - const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } }); - const request = httpServerMock.createKibanaRequest({ - headers: { authorization: header }, - routeTags: [ROUTE_TAG_ACCEPT_JWT], - }); - - const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); - mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); - mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); - mockOptions.client.asScoped.mockClear(); - - const provider = new HTTPAuthenticationProvider(mockOptions, { - supportedSchemes: new Set(['bearer']), - jwt: { taggedRoutesOnly: true }, - }); - - await expect(provider.authenticate(request)).resolves.toEqual( - AuthenticationResult.succeeded({ - ...user, - authentication_provider: { type: 'http', name: 'http' }, - }) - ); - - expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); - - expect(request.headers.authorization).toBe(header); - }); - - it('fails for JWT authentication if restricted to tagged routes and route is NOT tagged.', async () => { - const header = 'Bearer header.body.signature'; - const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } }); - const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } }); - - const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); - mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); - mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); - mockOptions.client.asScoped.mockClear(); - - const provider = new HTTPAuthenticationProvider(mockOptions, { - supportedSchemes: new Set(['bearer']), - jwt: { taggedRoutesOnly: true }, - }); - - await expect(provider.authenticate(request)).resolves.toEqual( - AuthenticationResult.notHandled() - ); - - expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); - - expect(request.headers.authorization).toBe(header); - }); - it('fails if authentication via `authorization` header with supported scheme fails.', async () => { const failureReason = new errors.ResponseError(securityMock.createApiResponse({ body: {} })); for (const { schemes, header } of [ diff --git a/x-pack/plugins/security/server/authentication/providers/http.ts b/x-pack/plugins/security/server/authentication/providers/http.ts index dd4bf8c40a43..21c2b25d3be8 100644 --- a/x-pack/plugins/security/server/authentication/providers/http.ts +++ b/x-pack/plugins/security/server/authentication/providers/http.ts @@ -9,22 +9,12 @@ import type { KibanaRequest } from '@kbn/core/server'; import type { AuthenticationProviderOptions } from './base'; import { BaseAuthenticationProvider } from './base'; -import { ROUTE_TAG_ACCEPT_JWT } from '../../routes/tags'; import { AuthenticationResult } from '../authentication_result'; import { DeauthenticationResult } from '../deauthentication_result'; import { HTTPAuthorizationHeader } from '../http_authentication'; -/** - * A type-string of the Elasticsearch JWT realm. - */ -const JWT_REALM_TYPE = 'jwt'; - interface HTTPAuthenticationProviderOptions { supportedSchemes: Set; - jwt?: { - // When set, only routes marked with `ROUTE_TAG_ACCEPT_JWT` tag will accept JWT as a means of authentication. - taggedRoutesOnly: boolean; - }; } /** @@ -42,11 +32,6 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider { */ private readonly supportedSchemes: Set; - /** - * Options relevant to the JWT authentication. - */ - private readonly jwt: HTTPAuthenticationProviderOptions['jwt']; - constructor( protected readonly options: Readonly, httpOptions: Readonly @@ -59,7 +44,6 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider { this.supportedSchemes = new Set( [...httpOptions.supportedSchemes].map((scheme) => scheme.toLowerCase()) ); - this.jwt = httpOptions.jwt; } /** @@ -95,23 +79,6 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider { this.logger.debug( `Request to ${request.url.pathname}${request.url.search} has been authenticated via authorization header with "${authorizationHeader.scheme}" scheme.` ); - - // If Kibana is configured to restrict JWT authentication only to selected routes, ensure that the route is marked - // with the `ROUTE_TAG_ACCEPT_JWT` tag to bypass that restriction. - if ( - user.authentication_realm.type === JWT_REALM_TYPE && - this.jwt?.taggedRoutesOnly && - !request.route.options.tags.includes(ROUTE_TAG_ACCEPT_JWT) - ) { - // Log a portion of the JWT signature to make debugging easier. - const jwtExcerpt = authorizationHeader.credentials.slice(-10); - this.logger.error( - `Attempted to authenticate with JWT credentials (…${jwtExcerpt}) against ${request.url.pathname}${request.url.search}, but it's not allowed. ` + - `Ensure that the route is defined with the "${ROUTE_TAG_ACCEPT_JWT}" tag.` - ); - return AuthenticationResult.notHandled(); - } - return AuthenticationResult.succeeded(user); } catch (err) { this.logger.debug( diff --git a/x-pack/plugins/security/server/config.test.ts b/x-pack/plugins/security/server/config.test.ts index 04b16aebab9c..47b16b575279 100644 --- a/x-pack/plugins/security/server/config.test.ts +++ b/x-pack/plugins/security/server/config.test.ts @@ -183,68 +183,6 @@ describe('config schema', () => { "showNavLinks": true, } `); - - expect(ConfigSchema.validate({}, { serverless: true, dist: true })).toMatchInlineSnapshot(` - Object { - "audit": Object { - "enabled": false, - }, - "authc": Object { - "http": Object { - "autoSchemesEnabled": true, - "enabled": true, - "jwt": Object { - "taggedRoutesOnly": true, - }, - "schemes": Array [ - "apikey", - "bearer", - ], - }, - "providers": Object { - "anonymous": undefined, - "basic": Object { - "basic": Object { - "accessAgreement": undefined, - "description": undefined, - "enabled": true, - "hint": undefined, - "icon": undefined, - "order": 0, - "session": Object { - "idleTimeout": undefined, - "lifespan": undefined, - }, - "showInSelector": true, - }, - }, - "kerberos": undefined, - "oidc": undefined, - "pki": undefined, - "saml": undefined, - "token": undefined, - }, - "selector": Object {}, - }, - "cookieName": "sid", - "enabled": true, - "loginAssistanceMessage": "", - "public": Object {}, - "secureCookies": false, - "session": Object { - "cleanupInterval": "PT1H", - "idleTimeout": "P3D", - "lifespan": "P30D", - }, - "showInsecureClusterWarning": true, - "showNavLinks": true, - "ui": Object { - "roleManagementEnabled": true, - "roleMappingManagementEnabled": true, - "userManagementEnabled": true, - }, - } - `); }); it('should throw error if xpack.security.encryptionKey is less than 32 characters', () => { @@ -1474,34 +1412,6 @@ describe('config schema', () => { }); }); - describe('authc.http', () => { - it('should not allow xpack.security.authc.http.jwt.* to be configured outside of the serverless context', () => { - expect(() => - ConfigSchema.validate( - { authc: { http: { jwt: { taggedRoutesOnly: false } } } }, - { serverless: false } - ) - ).toThrowErrorMatchingInlineSnapshot( - `"[authc.http.jwt]: a value wasn't expected to be present"` - ); - }); - - it('should allow xpack.security.authc.http.jwt.* to be configured inside of the serverless context', () => { - expect( - ConfigSchema.validate( - { authc: { http: { jwt: { taggedRoutesOnly: false } } } }, - { serverless: true } - ).ui - ).toMatchInlineSnapshot(` - Object { - "roleManagementEnabled": true, - "roleMappingManagementEnabled": true, - "userManagementEnabled": true, - } - `); - }); - }); - describe('ui', () => { it('should not allow xpack.security.ui.* to be configured outside of the serverless context', () => { expect(() => diff --git a/x-pack/plugins/security/server/config.ts b/x-pack/plugins/security/server/config.ts index a5483b4e70ba..15cb6461b97b 100644 --- a/x-pack/plugins/security/server/config.ts +++ b/x-pack/plugins/security/server/config.ts @@ -279,11 +279,6 @@ export const ConfigSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), autoSchemesEnabled: schema.boolean({ defaultValue: true }), schemes: schema.arrayOf(schema.string(), { defaultValue: ['apikey', 'bearer'] }), - jwt: offeringBasedSchema({ - serverless: schema.object({ - taggedRoutesOnly: schema.boolean({ defaultValue: true }), - }), - }), }), }), audit: schema.object({ diff --git a/x-pack/plugins/security/server/routes/tags.ts b/x-pack/plugins/security/server/routes/tags.ts index a6ffd49d53a5..090c04d29757 100644 --- a/x-pack/plugins/security/server/routes/tags.ts +++ b/x-pack/plugins/security/server/routes/tags.ts @@ -25,9 +25,3 @@ export const ROUTE_TAG_CAN_REDIRECT = 'security:canRedirect'; * parties, require special handling. */ export const ROUTE_TAG_AUTH_FLOW = 'security:authFlow'; - -/** - * If `xpack.security.authc.http.jwt.taggedRoutesOnly` flag is set, then only routes marked with this tag will accept - * JWT as a means of authentication. - */ -export const ROUTE_TAG_ACCEPT_JWT = 'security:acceptJWT'; diff --git a/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts b/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts index 322060b4f9b6..e70c78b8e212 100644 --- a/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts +++ b/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts @@ -57,13 +57,11 @@ describe('backgroundTaskUtilizationRoute', () => { `"/internal/task_manager/_background_task_utilization"` ); expect(config1.options?.authRequired).toEqual(true); - expect(config1.options?.tags).toEqual(['security:acceptJWT']); const [config2] = router.get.mock.calls[1]; expect(config2.path).toMatchInlineSnapshot(`"/api/task_manager/_background_task_utilization"`); expect(config2.options?.authRequired).toEqual(true); - expect(config2.options?.tags).toEqual(['security:acceptJWT']); }); it(`sets "authRequired" to false when config.unsafe.authenticate_background_task_utilization is set to false`, async () => { diff --git a/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts b/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts index b72b8ad5a704..38b1ce9966f3 100644 --- a/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts +++ b/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts @@ -117,9 +117,6 @@ export function backgroundTaskUtilizationRoute( options: { access: 'public', // access must be public to allow "system" users, like metrics collectors, to access these routes authRequired: routeOption.isAuthenticated ?? true, - // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to - // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. - tags: ['security:acceptJWT'], }, }, async function ( diff --git a/x-pack/plugins/task_manager/server/routes/metrics.test.ts b/x-pack/plugins/task_manager/server/routes/metrics.test.ts index 172e29d61f11..a9703aa7548d 100644 --- a/x-pack/plugins/task_manager/server/routes/metrics.test.ts +++ b/x-pack/plugins/task_manager/server/routes/metrics.test.ts @@ -28,7 +28,6 @@ describe('metricsRoute', () => { const [config] = router.get.mock.calls[0]; expect(config.path).toMatchInlineSnapshot(`"/api/task_manager/metrics"`); - expect(config.options?.tags).toEqual(['security:acceptJWT']); }); it('emits resetMetric$ event when route is accessed and reset query param is true', async () => { diff --git a/x-pack/plugins/task_manager/server/routes/metrics.ts b/x-pack/plugins/task_manager/server/routes/metrics.ts index 692227899979..737f2b44fd79 100644 --- a/x-pack/plugins/task_manager/server/routes/metrics.ts +++ b/x-pack/plugins/task_manager/server/routes/metrics.ts @@ -48,9 +48,6 @@ export function metricsRoute(params: MetricsRouteParams) { path: `/api/task_manager/metrics`, options: { access: 'public', - // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to - // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. - tags: ['security:acceptJWT'], }, // Uncomment when we determine that we can restrict API usage to Global admins based on telemetry // options: { tags: ['access:taskManager'] }, diff --git a/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc b/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc index accbad462016..6eeec0c14c5c 100644 --- a/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc +++ b/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc @@ -1,6 +1,6 @@ { "type": "shared-common", "id": "@kbn/security-api-integration-helpers", - "owner": "@elastic/kibana-security", + "owner": "@elastic/kibana-core", "devOnly": true } diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index.ts index 4ea6b50bb8f2..3f6751b8e4d0 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index.ts @@ -13,7 +13,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./security/anonymous')); loadTestFile(require.resolve('./security/api_keys')); loadTestFile(require.resolve('./security/authentication')); - loadTestFile(require.resolve('./security/authentication_http')); loadTestFile(require.resolve('./security/authorization')); loadTestFile(require.resolve('./security/misc')); loadTestFile(require.resolve('./security/response_headers')); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts deleted file mode 100644 index 1555ebe352df..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertestWithoutAuth'); - - describe('security/authentication/http', function () { - it('allows JWT HTTP authentication only for selected routes', async () => { - const jsonWebToken = - 'eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwczovL2tpYmFuYS5lbGFzdGljLmNvL2p3dC8iLCJzdWIiOiJlbGFzdGljLWFnZW50IiwiYXVkIjoiZWxhc3RpY3NlYXJjaCIsIm5hbWUiOiJFbGFzdGljIEFnZW50IiwiaWF0Ijo5NDY2ODQ4MDAsImV4cCI6NDA3MDkwODgwMH0.P7RHKZlLskS5DfVRqoVO4ivoIq9rXl2-GW6hhC9NvTSkwphYivcjpTVcyENZvxTTvJJNqcyx6rF3T-7otTTIHBOZIMhZauc5dob-sqcN_mT2htqm3BpSdlJlz60TBq6diOtlNhV212gQCEJMPZj0MNj7kZRj_GsECrTaU7FU0A3HAzkbdx15vQJMKZiFbbQCVI7-X2J0bZzQKIWfMHD-VgHFwOe6nomT-jbYIXtCBDd6fNj1zTKRl-_uzjVqNK-h8YW1h6tE4xvZmXyHQ1-9yNKZIWC7iEaPkBLaBKQulLU5MvW3AtVDUhzm6--5H1J85JH5QhRrnKYRon7ZW5q1AQ'; - - // Check 5 routes that are currently known to accept JWT as a means of authentication. - for (const allowedPath of [ - '/api/status', - '/api/stats', - '/api/task_manager/_background_task_utilization', - '/internal/task_manager/_background_task_utilization', - '/api/task_manager/metrics', - ]) { - await supertest - .get(allowedPath) - .set('Authorization', `Bearer ${jsonWebToken}`) - .set('ES-Client-Authentication', 'SharedSecret my_super_secret') - .set(svlCommonApi.getInternalRequestHeader()) - .expect(200); - } - - // Make sure it's not possible to use JWT to have interactive sessions. - await supertest - .get('/') - .set('Authorization', `Bearer ${jsonWebToken}`) - .set('ES-Client-Authentication', 'SharedSecret my_super_secret') - .expect(401); - - // Make sure it's not possible to use JWT to access any other APIs. - await supertest - .get('/internal/security/me') - .set('Authorization', `Bearer ${jsonWebToken}`) - .set('ES-Client-Authentication', 'SharedSecret my_super_secret') - .set(svlCommonApi.getInternalRequestHeader()) - .expect(401); - }); - }); -} diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 89b08cfd34ea..40e59a747897 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -33,33 +33,14 @@ export default async () => { '../../test/security_api_integration/plugins/saml_provider' ); - const jwksPath = require.resolve('@kbn/security-api-integration-helpers/oidc/jwks.json'); - return { servers, esTestCluster: { from: 'serverless', serverArgs: [ - 'xpack.security.authc.realms.file.file1.order=-100', - - 'xpack.security.authc.realms.jwt.jwt1.order=-98', - `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`, - 'xpack.security.authc.realms.jwt.jwt1.client_authentication.type=shared_secret', - `xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret=my_super_secret`, - `xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/`, - `xpack.security.authc.realms.jwt.jwt1.allowed_subjects=elastic-agent`, - 'xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch', - `xpack.security.authc.realms.jwt.jwt1.allowed_signature_algorithms=[RS256]`, - `xpack.security.authc.realms.jwt.jwt1.claims.principal=sub`, - `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${jwksPath}`, - - // TODO: We should set this flag to `false` as soon as we fully migrate tests to SAML and file realms. - `xpack.security.authc.realms.native.native1.enabled=true`, - `xpack.security.authc.realms.native.native1.order=-97`, - 'xpack.security.authc.token.enabled=true', - 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=101', + 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${idpPath}`, 'xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=http://www.elastic.co/saml1', `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://localhost:${servers.kibana.port}`, From 7d782616102957f449a9ae4bb2b3733e16dfb859 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 24 Aug 2023 10:49:18 -0600 Subject: [PATCH 098/182] disable http ssl --- packages/kbn-test/src/es/test_es_cluster.ts | 2 +- x-pack/test_serverless/shared/config.base.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 8adb22b2724d..703027296d9b 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -225,7 +225,7 @@ export function createTestEsCluster< } else if (esFrom === 'serverless') { return await firstNode.runServerless({ basePath, - esArgs: ['xpack.security.http.ssl.enabled=false', ...customEsArgs], + esArgs: customEsArgs, port, clean: true, teardown: true, diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index dee8dd567359..cce278d47c9c 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -41,6 +41,9 @@ export default async () => { esTestCluster: { from: 'serverless', serverArgs: [ + // HTTP SSL requires setup for Kibana to trust ESS, disable for now + 'xpack.security.http.ssl.enabled=false', + 'xpack.security.authc.realms.file.file1.order=-100', 'xpack.security.authc.realms.jwt.jwt1.order=-98', From 5b6610591f575992bf484385c34c0503d65cc52f Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 24 Aug 2023 11:22:31 -0600 Subject: [PATCH 099/182] Update ess resources readme for adding users --- packages/kbn-es/src/ess_resources/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/ess_resources/README.md b/packages/kbn-es/src/ess_resources/README.md index 49118c5da7bb..f70f7a25ebb0 100644 --- a/packages/kbn-es/src/ess_resources/README.md +++ b/packages/kbn-es/src/ess_resources/README.md @@ -1,7 +1,9 @@ # Elasticsearch Serverless Resources The resources in this directory are used for seeding Elasticsearch Serverless (ESS) images with users, roles and tokens for SSL and authentication. ESS requires file realm authentication, so we will bind mount them into the containers at `/usr/share/elasticsearch/config/`. -## Default User +## Users + +### Default user The default superuser authentication to login to Kibana is: @@ -10,6 +12,13 @@ username: kibana_serverless_superuser password: changeme ``` +### Adding users + +1. Add the user:encrypted_password to `users` file. The encrypted password for `kibana_serverless_superuser` is `changeme` if you want to reuse the value. +1. Set the new user's roles in `users_roles` file. +1. Add the username to `operator_users.yml` in the array for file realm users. + + ## Service Account and Tokens This section for Service Accounts was originally from the [ESS repository](https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/resources/README.service_tokens.md). From 35e777e29e7e18611fcea4aae1f186e57967860e Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 24 Aug 2023 19:10:58 +0100 Subject: [PATCH 100/182] Revert "Prepare the Security domain HTTP APIs for Serverless (#162087)" This reverts commit fe0ffab1da904945ec2fc6c7106c7e9a8f4eed72. --- config/serverless.yml | 3 - .../encrypted_saved_objects/server/plugin.ts | 30 +-- x-pack/plugins/security/server/plugin.ts | 1 - .../server/routes/authentication/common.ts | 46 ++-- .../server/routes/authentication/saml.ts | 7 +- .../server/routes/authorization/index.ts | 12 +- .../plugins/security/server/routes/index.ts | 23 +- .../server/routes/session_management/index.ts | 9 +- .../server/routes/views/index.test.ts | 55 +---- .../security/server/routes/views/index.ts | 20 +- x-pack/plugins/spaces/server/plugin.ts | 25 +- .../routes/api/external/copy_to_space.test.ts | 2 +- .../routes/api/external/copy_to_space.ts | 6 +- .../server/routes/api/external/delete.test.ts | 2 +- .../server/routes/api/external/delete.ts | 4 +- .../disable_legacy_url_aliases.test.ts | 2 +- .../external/disable_legacy_url_aliases.ts | 4 +- .../server/routes/api/external/get.test.ts | 2 +- .../spaces/server/routes/api/external/get.ts | 4 +- .../routes/api/external/get_all.test.ts | 2 +- .../server/routes/api/external/get_all.ts | 4 +- .../external/get_shareable_references.test.ts | 2 +- .../api/external/get_shareable_references.ts | 4 +- .../server/routes/api/external/index.ts | 26 +- .../server/routes/api/external/post.test.ts | 2 +- .../spaces/server/routes/api/external/post.ts | 4 +- .../server/routes/api/external/put.test.ts | 2 +- .../spaces/server/routes/api/external/put.ts | 4 +- .../external/update_objects_spaces.test.ts | 2 +- .../api/external/update_objects_spaces.ts | 4 +- .../api/internal/get_active_space.test.ts | 2 +- .../routes/api/internal/get_active_space.ts | 4 +- .../server/routes/api/internal/index.ts | 2 +- .../spaces_client/spaces_client.test.ts | 18 +- x-pack/plugins/spaces/tsconfig.json | 1 - .../api_integration/services/index.ts | 2 - .../api_integration/services/saml_tools.ts | 42 ---- .../services/svl_common_api.ts | 9 - .../common/encrypted_saved_objects.ts | 26 -- .../test_suites/common/index.ts | 14 +- .../test_suites/common/security/anonymous.ts | 33 --- .../test_suites/common/security/api_keys.ts | 212 ---------------- .../common/security/authentication.ts | 201 ---------------- .../common/security/authorization.ts | 110 --------- .../test_suites/common/security/misc.ts | 58 ----- .../common/security/role_mappings.ts | 55 ----- .../test_suites/common/security/sessions.ts | 78 ------ .../common/security/user_profiles.ts | 48 ---- .../test_suites/common/security/users.ts | 132 ---------- .../test_suites/common/security/views.ts | 114 --------- ...eaders.ts => security_response_headers.ts} | 4 +- .../test_suites/common/security_users.ts | 27 +++ .../test_suites/common/spaces.ts | 226 +++--------------- .../security/cypress/tasks/login.ts | 3 +- x-pack/test_serverless/shared/config.base.ts | 32 +-- x-pack/test_serverless/tsconfig.json | 1 - 56 files changed, 186 insertions(+), 1581 deletions(-) delete mode 100644 x-pack/test_serverless/api_integration/services/saml_tools.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/users.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/views.ts rename x-pack/test_serverless/api_integration/test_suites/common/{security/response_headers.ts => security_response_headers.ts} (95%) create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security_users.ts diff --git a/config/serverless.yml b/config/serverless.yml index e18049e8517c..3b0587d45cae 100644 --- a/config/serverless.yml +++ b/config/serverless.yml @@ -50,9 +50,6 @@ xpack.cloud_integrations.data_migration.enabled: false data.search.sessions.enabled: false advanced_settings.enabled: false -# Disable the browser-side functionality that depends on SecurityCheckupGetStateRoutes -xpack.security.showInsecureClusterWarning: false - # Disable UI of security management plugins xpack.security.ui.userManagementEnabled: false xpack.security.ui.roleManagementEnabled: false diff --git a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts index 7b7fed43f518..92a663fab64d 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts @@ -95,23 +95,19 @@ export class EncryptedSavedObjectsPlugin getStartServices: core.getStartServices, }); - // In the serverless environment, the encryption keys for saved objects is managed internally and never - // exposed to users and administrators, eliminating the need for any public Encrypted Saved Objects HTTP APIs - if (this.initializerContext.env.packageInfo.buildFlavor !== 'serverless') { - defineRoutes({ - router: core.http.createRouter(), - logger: this.initializerContext.logger.get('routes'), - encryptionKeyRotationService: Object.freeze( - new EncryptionKeyRotationService({ - logger: this.logger.get('key-rotation-service'), - service, - getStartServices: core.getStartServices, - security: deps.security, - }) - ), - config, - }); - } + defineRoutes({ + router: core.http.createRouter(), + logger: this.initializerContext.logger.get('routes'), + encryptionKeyRotationService: Object.freeze( + new EncryptionKeyRotationService({ + logger: this.logger.get('key-rotation-service'), + service, + getStartServices: core.getStartServices, + security: deps.security, + }) + ), + config, + }); return { canEncrypt, diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index d196dd9f4113..4f36c0bf508d 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -359,7 +359,6 @@ export class SecurityPlugin getAnonymousAccessService: this.getAnonymousAccess, getUserProfileService: this.getUserProfileService, analyticsService: this.analyticsService.setup({ analytics: core.analytics }), - buildFlavor: this.initializerContext.env.packageInfo.buildFlavor, }); return Object.freeze({ diff --git a/x-pack/plugins/security/server/routes/authentication/common.ts b/x-pack/plugins/security/server/routes/authentication/common.ts index f359c320151e..4eeeed299809 100644 --- a/x-pack/plugins/security/server/routes/authentication/common.ts +++ b/x-pack/plugins/security/server/routes/authentication/common.ts @@ -32,14 +32,9 @@ export function defineCommonRoutes({ basePath, license, logger, - buildFlavor, }: RouteDefinitionParams) { // Generate two identical routes with new and deprecated URL and issue a warning if route with deprecated URL is ever used. - // For a serverless build, do not register deprecated versioned routes - for (const path of [ - '/api/security/logout', - ...(buildFlavor !== 'serverless' ? ['/api/security/v1/logout'] : []), - ]) { + for (const path of ['/api/security/logout', '/api/security/v1/logout']) { router.get( { path, @@ -84,11 +79,7 @@ export function defineCommonRoutes({ } // Generate two identical routes with new and deprecated URL and issue a warning if route with deprecated URL is ever used. - // For a serverless build, do not register deprecated versioned routes - for (const path of [ - '/internal/security/me', - ...(buildFlavor !== 'serverless' ? ['/api/security/v1/me'] : []), - ]) { + for (const path of ['/internal/security/me', '/api/security/v1/me']) { router.get( { path, validate: false }, createLicensedRouteHandler((context, request, response) => { @@ -132,8 +123,6 @@ export function defineCommonRoutes({ return undefined; } - // Register the login route for serverless for the time being. Note: This route will move into the buildFlavor !== 'serverless' block below. See next line. - // ToDo: In the serverless environment, we do not support API login - the only valid authentication methodology (or maybe just method or mechanism?) is SAML router.post( { path: '/internal/security/login', @@ -180,23 +169,20 @@ export function defineCommonRoutes({ }) ); - if (buildFlavor !== 'serverless') { - // In the serverless offering, the access agreement functionality isn't available. - router.post( - { path: '/internal/security/access_agreement/acknowledge', validate: false }, - createLicensedRouteHandler(async (context, request, response) => { - // If license doesn't allow access agreement we shouldn't handle request. - if (!license.getFeatures().allowAccessAgreement) { - logger.warn(`Attempted to acknowledge access agreement when license doesn't allow it.`); - return response.forbidden({ - body: { message: `Current license doesn't support access agreement.` }, - }); - } + router.post( + { path: '/internal/security/access_agreement/acknowledge', validate: false }, + createLicensedRouteHandler(async (context, request, response) => { + // If license doesn't allow access agreement we shouldn't handle request. + if (!license.getFeatures().allowAccessAgreement) { + logger.warn(`Attempted to acknowledge access agreement when license doesn't allow it.`); + return response.forbidden({ + body: { message: `Current license doesn't support access agreement.` }, + }); + } - await getAuthenticationService().acknowledgeAccessAgreement(request); + await getAuthenticationService().acknowledgeAccessAgreement(request); - return response.noContent(); - }) - ); - } + return response.noContent(); + }) + ); } diff --git a/x-pack/plugins/security/server/routes/authentication/saml.ts b/x-pack/plugins/security/server/routes/authentication/saml.ts index ddc31fbc88b8..350f3527f331 100644 --- a/x-pack/plugins/security/server/routes/authentication/saml.ts +++ b/x-pack/plugins/security/server/routes/authentication/saml.ts @@ -19,14 +19,9 @@ export function defineSAMLRoutes({ getAuthenticationService, basePath, logger, - buildFlavor, }: RouteDefinitionParams) { // Generate two identical routes with new and deprecated URL and issue a warning if route with deprecated URL is ever used. - // For a serverless build, do not register deprecated versioned routes - for (const path of [ - '/api/security/saml/callback', - ...(buildFlavor !== 'serverless' ? ['/api/security/v1/saml'] : []), - ]) { + for (const path of ['/api/security/saml/callback', '/api/security/v1/saml']) { router.post( { path, diff --git a/x-pack/plugins/security/server/routes/authorization/index.ts b/x-pack/plugins/security/server/routes/authorization/index.ts index 0e4cc467f3b1..b3b29e950d72 100644 --- a/x-pack/plugins/security/server/routes/authorization/index.ts +++ b/x-pack/plugins/security/server/routes/authorization/index.ts @@ -12,14 +12,8 @@ import { defineShareSavedObjectPermissionRoutes } from './spaces'; import type { RouteDefinitionParams } from '..'; export function defineAuthorizationRoutes(params: RouteDefinitionParams) { - // The reset session endpoint is registered with httpResources and should remain public in serverless + defineRolesRoutes(params); + definePrivilegesRoutes(params); resetSessionPageRoutes(params); - defineRolesRoutes(params); // Temporarily allow role APIs (ToDo: move to non-serverless block below) - - // In the serverless environment, roles, privileges, and permissions are managed internally and only - // exposed to users and administrators via control plane UI, eliminating the need for any public HTTP APIs. - if (params.buildFlavor !== 'serverless') { - definePrivilegesRoutes(params); - defineShareSavedObjectPermissionRoutes(params); - } + defineShareSavedObjectPermissionRoutes(params); } diff --git a/x-pack/plugins/security/server/routes/index.ts b/x-pack/plugins/security/server/routes/index.ts index 99739208e6b7..ba33ca319cd2 100644 --- a/x-pack/plugins/security/server/routes/index.ts +++ b/x-pack/plugins/security/server/routes/index.ts @@ -7,7 +7,6 @@ import type { Observable } from 'rxjs'; -import type { BuildFlavor } from '@kbn/config/src/types'; import type { HttpResources, IBasePath, Logger } from '@kbn/core/server'; import type { KibanaFeature } from '@kbn/features-plugin/server'; import type { PublicMethodsOf } from '@kbn/utility-types'; @@ -55,26 +54,20 @@ export interface RouteDefinitionParams { getUserProfileService: () => UserProfileServiceStartInternal; getAnonymousAccessService: () => AnonymousAccessServiceStart; analyticsService: AnalyticsServiceSetup; - buildFlavor: BuildFlavor; } export function defineRoutes(params: RouteDefinitionParams) { - defineAnalyticsRoutes(params); - defineApiKeysRoutes(params); defineAuthenticationRoutes(params); defineAuthorizationRoutes(params); defineSessionManagementRoutes(params); + defineApiKeysRoutes(params); + defineIndicesRoutes(params); + defineUsersRoutes(params); defineUserProfileRoutes(params); - defineUsersRoutes(params); // Temporarily allow user APIs (ToDo: move to non-serverless block below) + defineRoleMappingRoutes(params); defineViewRoutes(params); - - // In the serverless environment... - if (params.buildFlavor !== 'serverless') { - defineAnonymousAccessRoutes(params); // anonymous access is disabled - defineDeprecationsRoutes(params); // deprecated kibana user roles are not applicable, these HTTP APIs are not needed - defineIndicesRoutes(params); // the ES privileges form used to help define roles (only consumer) is disabled, so there is no need for these HTTP APIs - defineRoleMappingRoutes(params); // role mappings are managed internally, based on configurations in control plane, these HTTP APIs are not needed - defineSecurityCheckupGetStateRoutes(params); // security checkup is not applicable, these HTTP APIs are not needed - // defineUsersRoutes(params); // the native realm is not enabled (there is only Elastic cloud SAML), no user HTTP API routes are needed - } + defineDeprecationsRoutes(params); + defineAnonymousAccessRoutes(params); + defineSecurityCheckupGetStateRoutes(params); + defineAnalyticsRoutes(params); } diff --git a/x-pack/plugins/security/server/routes/session_management/index.ts b/x-pack/plugins/security/server/routes/session_management/index.ts index c095a7740997..041feea8a62f 100644 --- a/x-pack/plugins/security/server/routes/session_management/index.ts +++ b/x-pack/plugins/security/server/routes/session_management/index.ts @@ -13,12 +13,5 @@ import type { RouteDefinitionParams } from '..'; export function defineSessionManagementRoutes(params: RouteDefinitionParams) { defineSessionInfoRoutes(params); defineSessionExtendRoutes(params); - - // The invalidate session API was introduced to address situations where the session index - // could grow rapidly - when session timeouts are disabled, or with anonymous access. - // In the serverless environment, sessions timeouts are always be enabled, and there is no - // anonymous access. This eliminates the need for an invalidate session HTTP API. - if (params.buildFlavor !== 'serverless') { - defineInvalidateSessionsRoutes(params); - } + defineInvalidateSessionsRoutes(params); } diff --git a/x-pack/plugins/security/server/routes/views/index.test.ts b/x-pack/plugins/security/server/routes/views/index.test.ts index f5f0296a39c1..755a5a1202c7 100644 --- a/x-pack/plugins/security/server/routes/views/index.test.ts +++ b/x-pack/plugins/security/server/routes/views/index.test.ts @@ -12,7 +12,6 @@ describe('View routes', () => { it('does not register Login routes if both `basic` and `token` providers are disabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { providers: { pki: { pki1: { order: 0 } } } }, - accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -20,12 +19,12 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ + "/security/access_agreement", "/security/account", - "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/security/access_agreement", + "/internal/security/capture-url", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` @@ -38,7 +37,6 @@ describe('View routes', () => { it('registers Login routes if `basic` provider is enabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { providers: { basic: { basic1: { order: 0 } } } }, - accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -46,19 +44,19 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ + "/login", + "/security/access_agreement", "/security/account", - "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/security/access_agreement", - "/login", + "/internal/security/capture-url", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` Array [ - "/internal/security/access_agreement/state", "/internal/security/login_state", + "/internal/security/access_agreement/state", ] `); }); @@ -66,7 +64,6 @@ describe('View routes', () => { it('registers Login routes if `token` provider is enabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { providers: { token: { token1: { order: 0 } } } }, - accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -74,19 +71,19 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ + "/login", + "/security/access_agreement", "/security/account", - "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/security/access_agreement", - "/login", + "/internal/security/capture-url", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` Array [ - "/internal/security/access_agreement/state", "/internal/security/login_state", + "/internal/security/access_agreement/state", ] `); }); @@ -94,7 +91,6 @@ describe('View routes', () => { it('registers Login routes if Login Selector is enabled even if both `token` and `basic` providers are not enabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { selector: { enabled: true }, providers: { pki: { pki1: { order: 0 } } } }, - accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -102,44 +98,19 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ - "/security/account", - "/internal/security/capture-url", - "/security/logged_out", - "/logout", - "/security/overwritten_session", - "/security/access_agreement", "/login", - ] - `); - expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` - Array [ - "/internal/security/access_agreement/state", - "/internal/security/login_state", - ] - `); - }); - - it('does not register access agreement routes if access agreement is not enabled', () => { - const routeParamsMock = routeDefinitionParamsMock.create({ - authc: { providers: { basic: { basic1: { order: 0 } } } }, - }); - - defineViewRoutes(routeParamsMock); - - expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) - .toMatchInlineSnapshot(` - Array [ + "/security/access_agreement", "/security/account", - "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/login", + "/internal/security/capture-url", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` Array [ "/internal/security/login_state", + "/internal/security/access_agreement/state", ] `); }); diff --git a/x-pack/plugins/security/server/routes/views/index.ts b/x-pack/plugins/security/server/routes/views/index.ts index c9fbb3b1bc36..f1efa4611dc5 100644 --- a/x-pack/plugins/security/server/routes/views/index.ts +++ b/x-pack/plugins/security/server/routes/views/index.ts @@ -15,23 +15,17 @@ import { defineOverwrittenSessionRoutes } from './overwritten_session'; import type { RouteDefinitionParams } from '..'; export function defineViewRoutes(params: RouteDefinitionParams) { - defineAccountManagementRoutes(params); - defineCaptureURLRoutes(params); - defineLoggedOutRoutes(params); - defineLogoutRoutes(params); - defineOverwrittenSessionRoutes(params); - - if ( - params.config.accessAgreement?.message || - params.config.authc.sortedProviders.some(({ hasAccessAgreement }) => hasAccessAgreement) - ) { - defineAccessAgreementRoutes(params); - } - if ( params.config.authc.selector.enabled || params.config.authc.sortedProviders.some(({ type }) => type === 'basic' || type === 'token') ) { defineLoginRoutes(params); } + + defineAccessAgreementRoutes(params); + defineAccountManagementRoutes(params); + defineLoggedOutRoutes(params); + defineLogoutRoutes(params); + defineOverwrittenSessionRoutes(params); + defineCaptureURLRoutes(params); } diff --git a/x-pack/plugins/spaces/server/plugin.ts b/x-pack/plugins/spaces/server/plugin.ts index 893d3db22d70..cb8b42f343ba 100644 --- a/x-pack/plugins/spaces/server/plugin.ts +++ b/x-pack/plugins/spaces/server/plugin.ts @@ -103,7 +103,7 @@ export class SpacesPlugin private defaultSpaceService?: DefaultSpaceService; - constructor(private readonly initializerContext: PluginInitializerContext) { + constructor(initializerContext: PluginInitializerContext) { this.config$ = initializerContext.config.create(); this.log = initializerContext.logger.get(); this.spacesService = new SpacesService(); @@ -148,21 +148,18 @@ export class SpacesPlugin logger: this.log, }); - const router = core.http.createRouter(); - - initExternalSpacesApi( - { - router, - log: this.log, - getStartServices: core.getStartServices, - getSpacesService, - usageStatsServicePromise, - }, - this.initializerContext.env.packageInfo.buildFlavor - ); + const externalRouter = core.http.createRouter(); + initExternalSpacesApi({ + externalRouter, + log: this.log, + getStartServices: core.getStartServices, + getSpacesService, + usageStatsServicePromise, + }); + const internalRouter = core.http.createRouter(); initInternalSpacesApi({ - router, + internalRouter, getSpacesService, }); diff --git a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts index 772231679907..4018cfa11b7b 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts @@ -80,7 +80,7 @@ describe('copy to space', () => { }); initCopyToSpacesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts index 0c866de5bde6..7faf03ea60b5 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts @@ -24,10 +24,10 @@ const areObjectsUnique = (objects: SavedObjectIdentifier[]) => _.uniqBy(objects, (o: SavedObjectIdentifier) => `${o.type}:${o.id}`).length === objects.length; export function initCopyToSpacesApi(deps: ExternalRouteDeps) { - const { router, getSpacesService, usageStatsServicePromise, getStartServices } = deps; + const { externalRouter, getSpacesService, usageStatsServicePromise, getStartServices } = deps; const usageStatsClientPromise = usageStatsServicePromise.then(({ getClient }) => getClient()); - router.post( + externalRouter.post( { path: '/api/spaces/_copy_saved_objects', options: { @@ -137,7 +137,7 @@ export function initCopyToSpacesApi(deps: ExternalRouteDeps) { }) ); - router.post( + externalRouter.post( { path: '/api/spaces/_resolve_copy_saved_objects_errors', options: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts b/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts index 09ed757a5df7..02792389424d 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts @@ -62,7 +62,7 @@ describe('Spaces Public API', () => { }); initDeleteSpacesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/delete.ts b/x-pack/plugins/spaces/server/routes/api/external/delete.ts index 63679f49847b..b2f0d71d34a2 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/delete.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/delete.ts @@ -15,9 +15,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initDeleteSpacesApi(deps: ExternalRouteDeps) { - const { router, log, getSpacesService } = deps; + const { externalRouter, log, getSpacesService } = deps; - router.delete( + externalRouter.delete( { path: '/api/spaces/space/{id}', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts index ed7c403182ab..6af58c124be0 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts @@ -65,7 +65,7 @@ describe('_disable_legacy_url_aliases', () => { }); initDisableLegacyUrlAliasesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts index 725d0c87612f..ff523ea99c06 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts @@ -12,10 +12,10 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initDisableLegacyUrlAliasesApi(deps: ExternalRouteDeps) { - const { router, getSpacesService, usageStatsServicePromise } = deps; + const { externalRouter, getSpacesService, usageStatsServicePromise } = deps; const usageStatsClientPromise = usageStatsServicePromise.then(({ getClient }) => getClient()); - router.post( + externalRouter.post( { path: '/api/spaces/_disable_legacy_url_aliases', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/get.test.ts b/x-pack/plugins/spaces/server/routes/api/external/get.test.ts index 34c1ed01e0ce..43ac45ec3c4c 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get.test.ts @@ -61,7 +61,7 @@ describe('GET space', () => { }); initGetSpaceApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/get.ts b/x-pack/plugins/spaces/server/routes/api/external/get.ts index ce89aac5fe18..a26cfe0211d1 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get.ts @@ -13,9 +13,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetSpaceApi(deps: ExternalRouteDeps) { - const { router, getSpacesService } = deps; + const { externalRouter, getSpacesService } = deps; - router.get( + externalRouter.get( { path: '/api/spaces/space/{id}', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts b/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts index 93eab7a52b81..8fa87bf5ffa4 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts @@ -62,7 +62,7 @@ describe('GET /spaces/space', () => { }); initGetAllSpacesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_all.ts b/x-pack/plugins/spaces/server/routes/api/external/get_all.ts index c1dc24caf151..06d629a19456 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_all.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_all.ts @@ -13,9 +13,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetAllSpacesApi(deps: ExternalRouteDeps) { - const { router, log, getSpacesService } = deps; + const { externalRouter, log, getSpacesService } = deps; - router.get( + externalRouter.get( { path: '/api/spaces/space', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts index c9ea0d71c11b..daa957c04d11 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts @@ -61,7 +61,7 @@ describe('get shareable references', () => { spacesClientService: clientServiceStart, }); initGetShareableReferencesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts index f9b18961fae5..ec5b0ce82ece 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts @@ -12,9 +12,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetShareableReferencesApi(deps: ExternalRouteDeps) { - const { router, getStartServices } = deps; + const { externalRouter, getStartServices } = deps; - router.post( + externalRouter.post( { path: '/api/spaces/_get_shareable_references', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/index.ts b/x-pack/plugins/spaces/server/routes/api/external/index.ts index 290807f9b5f4..8716f63a5657 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/index.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { BuildFlavor } from '@kbn/config/src/types'; import type { CoreSetup, Logger } from '@kbn/core/server'; import { initCopyToSpacesApi } from './copy_to_space'; @@ -22,28 +21,21 @@ import type { SpacesRouter } from '../../../types'; import type { UsageStatsServiceSetup } from '../../../usage_stats'; export interface ExternalRouteDeps { - router: SpacesRouter; + externalRouter: SpacesRouter; getStartServices: CoreSetup['getStartServices']; getSpacesService: () => SpacesServiceStart; usageStatsServicePromise: Promise; log: Logger; } -export function initExternalSpacesApi(deps: ExternalRouteDeps, buildFlavor: BuildFlavor) { - // These two routes are always registered, internal in serverless by default +export function initExternalSpacesApi(deps: ExternalRouteDeps) { + initDeleteSpacesApi(deps); initGetSpaceApi(deps); initGetAllSpacesApi(deps); - - // In the serverless environment, Spaces are enabled but are effectively hidden from the user. We - // do not support more than 1 space: the default space. These HTTP APIs for creating, deleting, - // updating, and manipulating saved objects across multiple spaces are not needed. - if (buildFlavor !== 'serverless') { - initPutSpacesApi(deps); - initDeleteSpacesApi(deps); - initPostSpacesApi(deps); - initCopyToSpacesApi(deps); - initUpdateObjectsSpacesApi(deps); - initGetShareableReferencesApi(deps); - initDisableLegacyUrlAliasesApi(deps); - } + initPostSpacesApi(deps); + initPutSpacesApi(deps); + initCopyToSpacesApi(deps); + initUpdateObjectsSpacesApi(deps); + initGetShareableReferencesApi(deps); + initDisableLegacyUrlAliasesApi(deps); } diff --git a/x-pack/plugins/spaces/server/routes/api/external/post.test.ts b/x-pack/plugins/spaces/server/routes/api/external/post.test.ts index 88c763f31c0b..01c08eca85ec 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/post.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/post.test.ts @@ -62,7 +62,7 @@ describe('Spaces Public API', () => { }); initPostSpacesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/post.ts b/x-pack/plugins/spaces/server/routes/api/external/post.ts index d8091a0140e0..3ea6da647b4f 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/post.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/post.ts @@ -15,9 +15,9 @@ import { spaceSchema } from '../../../lib/space_schema'; import { createLicensedRouteHandler } from '../../lib'; export function initPostSpacesApi(deps: ExternalRouteDeps) { - const { router, log, getSpacesService } = deps; + const { externalRouter, log, getSpacesService } = deps; - router.post( + externalRouter.post( { path: '/api/spaces/space', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/put.test.ts b/x-pack/plugins/spaces/server/routes/api/external/put.test.ts index 5e1e6077a758..126d15268edf 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/put.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/put.test.ts @@ -62,7 +62,7 @@ describe('PUT /api/spaces/space', () => { }); initPutSpacesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/put.ts b/x-pack/plugins/spaces/server/routes/api/external/put.ts index 753ec8e02892..fb9f81857658 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/put.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/put.ts @@ -15,9 +15,9 @@ import { spaceSchema } from '../../../lib/space_schema'; import { createLicensedRouteHandler } from '../../lib'; export function initPutSpacesApi(deps: ExternalRouteDeps) { - const { router, getSpacesService } = deps; + const { externalRouter, getSpacesService } = deps; - router.put( + externalRouter.put( { path: '/api/spaces/space/{id}', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts index 195db8148a37..4c5fd44a6bb3 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts @@ -62,7 +62,7 @@ describe('update_objects_spaces', () => { spacesClientService: clientServiceStart, }); initUpdateObjectsSpacesApi({ - router, + externalRouter: router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts index 5e610c9693ab..ea95557514d5 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts @@ -14,7 +14,7 @@ import { SPACE_ID_REGEX } from '../../../lib/space_schema'; import { createLicensedRouteHandler } from '../../lib'; export function initUpdateObjectsSpacesApi(deps: ExternalRouteDeps) { - const { router, getStartServices } = deps; + const { externalRouter, getStartServices } = deps; const spacesSchema = schema.arrayOf( schema.string({ @@ -33,7 +33,7 @@ export function initUpdateObjectsSpacesApi(deps: ExternalRouteDeps) { } ); - router.post( + externalRouter.post( { path: '/api/spaces/_update_objects_spaces', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts index afa0f0a99594..172f1afec53c 100644 --- a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts @@ -26,7 +26,7 @@ describe('GET /internal/spaces/_active_space', () => { }); initGetActiveSpaceApi({ - router, + internalRouter: router, getSpacesService: () => service.start({ basePath: coreStart.http.basePath, diff --git a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts index 2996e7dbc4ed..feb6d8b59628 100644 --- a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts +++ b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts @@ -10,9 +10,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetActiveSpaceApi(deps: InternalRouteDeps) { - const { router, getSpacesService } = deps; + const { internalRouter, getSpacesService } = deps; - router.get( + internalRouter.get( { path: '/internal/spaces/_active_space', validate: false, diff --git a/x-pack/plugins/spaces/server/routes/api/internal/index.ts b/x-pack/plugins/spaces/server/routes/api/internal/index.ts index 0cf8135d7b71..2f732bfcaf5a 100644 --- a/x-pack/plugins/spaces/server/routes/api/internal/index.ts +++ b/x-pack/plugins/spaces/server/routes/api/internal/index.ts @@ -10,7 +10,7 @@ import type { SpacesServiceStart } from '../../../spaces_service/spaces_service' import type { SpacesRouter } from '../../../types'; export interface InternalRouteDeps { - router: SpacesRouter; + internalRouter: SpacesRouter; getSpacesService: () => SpacesServiceStart; } diff --git a/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts b/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts index 67c926229601..e9305f07f1b0 100644 --- a/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts +++ b/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts @@ -18,11 +18,7 @@ const createMockDebugLogger = () => { }; const createMockConfig = ( - mockConfig: ConfigType = { - enabled: true, - maxSpaces: 1000, - allowFeatureVisibility: true, - } + mockConfig: ConfigType = { enabled: true, maxSpaces: 1000, allowFeatureVisibility: true } ) => { return ConfigSchema.validate(mockConfig, { serverless: !mockConfig.allowFeatureVisibility }); }; @@ -213,11 +209,7 @@ describe('#create', () => { total: maxSpaces - 1, } as any); - const mockConfig = createMockConfig({ - enabled: true, - maxSpaces, - allowFeatureVisibility: true, - }); + const mockConfig = createMockConfig({ enabled: true, maxSpaces, allowFeatureVisibility: true }); const client = new SpacesClient(mockDebugLogger, mockConfig, mockCallWithRequestRepository, []); @@ -243,11 +235,7 @@ describe('#create', () => { total: maxSpaces, } as any); - const mockConfig = createMockConfig({ - enabled: true, - maxSpaces, - allowFeatureVisibility: true, - }); + const mockConfig = createMockConfig({ enabled: true, maxSpaces, allowFeatureVisibility: true }); const client = new SpacesClient(mockDebugLogger, mockConfig, mockCallWithRequestRepository, []); diff --git a/x-pack/plugins/spaces/tsconfig.json b/x-pack/plugins/spaces/tsconfig.json index d59e30e3a019..43ea0f1c6c56 100644 --- a/x-pack/plugins/spaces/tsconfig.json +++ b/x-pack/plugins/spaces/tsconfig.json @@ -31,7 +31,6 @@ "@kbn/core-custom-branding-browser-mocks", "@kbn/core-custom-branding-common", "@kbn/shared-ux-link-redirect-app", - "@kbn/config", ], "exclude": [ "target/**/*", diff --git a/x-pack/test_serverless/api_integration/services/index.ts b/x-pack/test_serverless/api_integration/services/index.ts index 06e7c33fd709..8102eeb9f4c1 100644 --- a/x-pack/test_serverless/api_integration/services/index.ts +++ b/x-pack/test_serverless/api_integration/services/index.ts @@ -12,7 +12,6 @@ import { services as svlSharedServices } from '../../shared/services'; import { SvlCommonApiServiceProvider } from './svl_common_api'; import { AlertingApiProvider } from './alerting_api'; -import { SamlToolsProvider } from './saml_tools'; import { DataViewApiProvider } from './data_view_api'; export const services = { @@ -21,7 +20,6 @@ export const services = { svlCommonApi: SvlCommonApiServiceProvider, alertingApi: AlertingApiProvider, - samlTools: SamlToolsProvider, dataViewApi: DataViewApiProvider, }; diff --git a/x-pack/test_serverless/api_integration/services/saml_tools.ts b/x-pack/test_serverless/api_integration/services/saml_tools.ts deleted file mode 100644 index 4756109fc667..000000000000 --- a/x-pack/test_serverless/api_integration/services/saml_tools.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -// eslint-disable-next-line @kbn/imports/no_boundary_crossing -import { getSAMLResponse } from '@kbn/security-api-integration-helpers/saml/saml_tools'; -import { kbnTestConfig } from '@kbn/test'; - -import { parse as parseCookie } from 'tough-cookie'; - -import { FtrProviderContext } from '../ftr_provider_context'; - -export function SamlToolsProvider({ getService }: FtrProviderContext) { - const supertestWithoutAuth = getService('supertestWithoutAuth'); - const randomness = getService('randomness'); - const svlCommonApi = getService('svlCommonApi'); - - function createSAMLResponse(options = {}) { - return getSAMLResponse({ - destination: `http://localhost:${kbnTestConfig.getPort()}/api/security/saml/callback`, - sessionIndex: String(randomness.naturalNumber()), - ...options, - }); - } - - return { - async login(username: string) { - const samlAuthenticationResponse = await supertestWithoutAuth - .post('/api/security/saml/callback') - .set(svlCommonApi.getCommonRequestHeader()) - .send({ SAMLResponse: await createSAMLResponse({ username }) }); - expect(samlAuthenticationResponse.status).to.equal(302); - expect(samlAuthenticationResponse.header.location).to.equal('/'); - const sessionCookie = parseCookie(samlAuthenticationResponse.header['set-cookie'][0])!; - return { Cookie: sessionCookie.cookieString() }; - }, - }; -} diff --git a/x-pack/test_serverless/api_integration/services/svl_common_api.ts b/x-pack/test_serverless/api_integration/services/svl_common_api.ts index b23c8f70a309..15b4f15f851f 100644 --- a/x-pack/test_serverless/api_integration/services/svl_common_api.ts +++ b/x-pack/test_serverless/api_integration/services/svl_common_api.ts @@ -36,14 +36,5 @@ export function SvlCommonApiServiceProvider({}: FtrProviderContext) { )}'` ); }, - - assertApiNotFound(body: unknown, status: number) { - expect(body).to.eql({ - statusCode: 404, - error: 'Not Found', - message: 'Not Found', - }); - expect(status).to.eql(404); - }, }; } diff --git a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts deleted file mode 100644 index be5dc924c839..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('encrypted saved objects', function () { - describe('route access', () => { - describe('disabled', () => { - it('rotate key', async () => { - const { body, status } = await supertest - .post('/api/encrypted_saved_objects/_rotate_key') - .set(svlCommonApi.getCommonRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index.ts index 3f6751b8e4d0..8296dd0d33c2 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index.ts @@ -9,19 +9,9 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('serverless common API', function () { - loadTestFile(require.resolve('./encrypted_saved_objects')); - loadTestFile(require.resolve('./security/anonymous')); - loadTestFile(require.resolve('./security/api_keys')); - loadTestFile(require.resolve('./security/authentication')); - loadTestFile(require.resolve('./security/authorization')); - loadTestFile(require.resolve('./security/misc')); - loadTestFile(require.resolve('./security/response_headers')); - loadTestFile(require.resolve('./security/role_mappings')); - loadTestFile(require.resolve('./security/sessions')); - loadTestFile(require.resolve('./security/users')); - loadTestFile(require.resolve('./security/user_profiles')); - loadTestFile(require.resolve('./security/views')); + loadTestFile(require.resolve('./security_users')); loadTestFile(require.resolve('./spaces')); + loadTestFile(require.resolve('./security_response_headers')); loadTestFile(require.resolve('./rollups')); loadTestFile(require.resolve('./scripted_fields')); loadTestFile(require.resolve('./index_management')); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts deleted file mode 100644 index 0b08aa18ce12..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/anonymous', function () { - describe('route access', () => { - describe('disabled', () => { - it('get access capabilities', async () => { - const { body, status } = await supertest - .get('/internal/security/anonymous_access/capabilities') - .set(svlCommonApi.getCommonRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get access state', async () => { - const { body, status } = await supertest - .get('/internal/security/anonymous_access/state') - .set(svlCommonApi.getCommonRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts deleted file mode 100644 index 256a0fcfe32a..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - let roleMapping: { id: string; name: string; api_key: string; encoded: string }; - - describe('security/api_keys', function () { - describe('route access', () => { - describe('internal', () => { - before(async () => { - const { body, status } = await supertest - .post('/internal/security/api_key') - .set(svlCommonApi.getInternalRequestHeader()) - .send({ - name: 'test', - metadata: {}, - role_descriptors: {}, - }); - expect(status).toBe(200); - roleMapping = body; - }); - - after(async () => { - const { body, status } = await supertest - .get('/internal/security/api_key?isAdmin=true') - .set(svlCommonApi.getInternalRequestHeader()); - - if (status === 200) { - await supertest - .post('/internal/security/api_key/invalidate') - .set(svlCommonApi.getInternalRequestHeader()) - .send({ - apiKeys: body?.apiKeys, - isAdmin: true, - }); - } - }); - - it('create', async () => { - let body: unknown; - let status: number; - const requestBody = { - name: 'create_test', - metadata: {}, - role_descriptors: {}, - }; - - ({ body, status } = await supertest - .post('/internal/security/api_key') - .set(svlCommonApi.getCommonRequestHeader()) - .send(requestBody)); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [post] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .post('/internal/security/api_key') - .set(svlCommonApi.getInternalRequestHeader()) - .send(requestBody)); - // expect success because we're using the internal header - expect(body).toEqual(expect.objectContaining({ name: 'create_test' })); - expect(status).toBe(200); - }); - - it('update', async () => { - let body: unknown; - let status: number; - const requestBody = { - id: roleMapping.id, - metadata: { test: 'value' }, - role_descriptors: {}, - }; - - ({ body, status } = await supertest - .put('/internal/security/api_key') - .set(svlCommonApi.getCommonRequestHeader()) - .send(requestBody)); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [put] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .put('/internal/security/api_key') - .set(svlCommonApi.getInternalRequestHeader()) - .send(requestBody)); - // expect success because we're using the internal header - expect(body).toEqual(expect.objectContaining({ updated: true })); - expect(status).toBe(200); - }); - - it('get all', async () => { - let body: unknown; - let status: number; - - ({ body, status } = await supertest - .get('/internal/security/api_key?isAdmin=true') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/internal/security/api_key?isAdmin=true') - .set(svlCommonApi.getInternalRequestHeader())); - // expect success because we're using the internal header - expect(body).toEqual( - expect.objectContaining({ - apiKeys: expect.arrayContaining([expect.objectContaining({ id: roleMapping.id })]), - }) - ); - expect(status).toBe(200); - }); - - it('get enabled', async () => { - let body: unknown; - let status: number; - - ({ body, status } = await supertest - .get('/internal/security/api_key/_enabled') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/internal/security/api_key/_enabled') - .set(svlCommonApi.getInternalRequestHeader())); - // expect success because we're using the internal header - expect(body).toEqual({ apiKeysEnabled: true }); - expect(status).toBe(200); - }); - - it('invalidate', async () => { - let body: unknown; - let status: number; - const requestBody = { - apiKeys: [ - { - id: roleMapping.id, - name: roleMapping.name, - }, - ], - isAdmin: true, - }; - - ({ body, status } = await supertest - .post('/internal/security/api_key/invalidate') - .set(svlCommonApi.getCommonRequestHeader()) - .send(requestBody)); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [post] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .post('/internal/security/api_key/invalidate') - .set(svlCommonApi.getInternalRequestHeader()) - .send(requestBody)); - // expect success because we're using the internal header - expect(body).toEqual({ - errors: [], - itemsInvalidated: [ - { - id: roleMapping.id, - name: roleMapping.name, - }, - ], - }); - expect(status).toBe(200); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts deleted file mode 100644 index 590adbe267b4..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/authentication', function () { - describe('route access', () => { - describe('disabled', () => { - // ToDo: uncomment when we disable login - // it('login', async () => { - // const { body, status } = await supertest - // .post('/internal/security/login') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - it('logout (deprecated)', async () => { - const { body, status } = await supertest - .get('/api/security/v1/logout') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get current user (deprecated)', async () => { - const { body, status } = await supertest - .get('/internal/security/v1/me') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('acknowledge access agreement', async () => { - const { body, status } = await supertest - .post('/internal/security/access_agreement/acknowledge') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - describe('OIDC', () => { - it('OIDC implicit', async () => { - const { body, status } = await supertest - .get('/api/security/oidc/implicit') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC implicit (deprecated)', async () => { - const { body, status } = await supertest - .get('/api/security/v1/oidc/implicit') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC implicit.js', async () => { - const { body, status } = await supertest - .get('/internal/security/oidc/implicit.js') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC callback', async () => { - const { body, status } = await supertest - .get('/api/security/oidc/callback') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC callback (deprecated)', async () => { - const { body, status } = await supertest - .get('/api/security/v1/oidc') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC login', async () => { - const { body, status } = await supertest - .post('/api/security/oidc/initiate_login') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC login (deprecated)', async () => { - const { body, status } = await supertest - .post('/api/security/v1/oidc') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('OIDC 3rd party login', async () => { - const { body, status } = await supertest - .get('/api/security/oidc/initiate_login') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - - it('SAML callback (deprecated)', async () => { - const { body, status } = await supertest - .post('/api/security/v1/saml') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - - describe('internal', () => { - it('get current user', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .get('/internal/security/me') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/internal/security/me') - .set(svlCommonApi.getInternalRequestHeader())); - // expect success because we're using the internal header - expect(body).toEqual({ - authentication_provider: { name: '__http__', type: 'http' }, - authentication_realm: { name: 'reserved', type: 'reserved' }, - authentication_type: 'realm', - elastic_cloud_user: false, - email: null, - enabled: true, - full_name: null, - lookup_realm: { name: 'reserved', type: 'reserved' }, - metadata: { _reserved: true }, - roles: ['superuser'], - username: 'elastic', - }); - expect(status).toBe(200); - }); - - // ToDo: remove when we disable login - it('login', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .post('/internal/security/login') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [post] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .post('/internal/security/login') - .set(svlCommonApi.getInternalRequestHeader())); - expect(status).not.toBe(404); - }); - }); - - describe('public', () => { - it('logout', async () => { - const { status } = await supertest.get('/api/security/logout'); - expect(status).toBe(302); - }); - - it('SAML callback', async () => { - const { body, status } = await supertest - .post('/api/security/saml/callback') - .set(svlCommonApi.getCommonRequestHeader()) - .send({ - SAMLResponse: '', - }); - - // Should fail with 401 (not 404) because there is no valid SAML response in the request body - expect(body).toEqual({ - error: 'Unauthorized', - message: 'Unauthorized', - statusCode: 401, - }); - expect(status).not.toBe(404); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts deleted file mode 100644 index d2f5db13da9b..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/authorization', function () { - describe('route access', () => { - describe('disabled', () => { - it('get all privileges', async () => { - const { body, status } = await supertest - .get('/api/security/privileges') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get built-in elasticsearch privileges', async () => { - const { body, status } = await supertest - .get('/internal/security/esPrivileges/builtin') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - // ToDo: Uncomment when we disable role APIs - // it('create/update role', async () => { - // const { body, status } = await supertest - // .put('/api/security/role/test') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('get role', async () => { - // const { body, status } = await supertest - // .get('/api/security/role/superuser') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('get all roles', async () => { - // const { body, status } = await supertest - // .get('/api/security/role') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('delete role', async () => { - // const { body, status } = await supertest - // .delete('/api/security/role/superuser') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - it('get shared saved object permissions', async () => { - const { body, status } = await supertest - .get('/internal/security/_share_saved_object_permissions') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - - // ToDo: remove when we disable role APIs - describe('internal', () => { - it('create/update role', async () => { - const { status } = await supertest - .put('/api/security/role/test') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('get role', async () => { - const { status } = await supertest - .get('/api/security/role/superuser') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('get all roles', async () => { - const { status } = await supertest - .get('/api/security/role') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('delete role', async () => { - const { status } = await supertest - .delete('/api/security/role/superuser') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - }); - - describe('public', () => { - it('reset session page', async () => { - const { status } = await supertest - .get('/internal/security/reset_session_page.js') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts deleted file mode 100644 index 2b49b3a96ab2..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/misc', function () { - describe('route access', () => { - describe('disabled', () => { - it('get index fields', async () => { - const { body, status } = await supertest - .get('/internal/security/fields/test') - .set(svlCommonApi.getInternalRequestHeader()) - .send({ params: 'params' }); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('fix deprecated roles', async () => { - const { body, status } = await supertest - .post('/internal/security/deprecations/kibana_user_role/_fix_users') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('fix deprecated role mappings', async () => { - const { body, status } = await supertest - .post('/internal/security/deprecations/kibana_user_role/_fix_role_mappings') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get security checkup state', async () => { - const { body, status } = await supertest - .get('/internal/security/security_checkup/state') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - - describe('internal', () => { - it('get record auth type', async () => { - const { status } = await supertest - .post('/internal/security/analytics/_record_auth_type') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).toBe(200); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts deleted file mode 100644 index 4d1f25cfd772..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/role_mappings', function () { - describe('route access', () => { - describe('disabled', () => { - it('create/update role mapping', async () => { - const { body, status } = await supertest - .post('/internal/security/role_mapping/test') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get role mapping', async () => { - const { body, status } = await supertest - .get('/internal/security/role_mapping/test') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get all role mappings', async () => { - const { body, status } = await supertest - .get('/internal/security/role_mapping') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('delete role mapping', async () => { - // this test works because the message for a missing endpoint is different from a missing role mapping - const { body, status } = await supertest - .delete('/internal/security/role_mapping/test') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('role mapping feature check', async () => { - const { body, status } = await supertest - .get('/internal/security/_check_role_mapping_features') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts deleted file mode 100644 index 2148447fa736..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/sessions', function () { - describe('route access', () => { - describe('disabled', () => { - it('invalidate', async () => { - const { body, status } = await supertest - .post('/api/security/session/_invalidate') - .set(svlCommonApi.getInternalRequestHeader()) - .send({ match: 'all' }); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - - describe('internal', () => { - it('get session info', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .get('/internal/security/session') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/internal/security/session') - .set(svlCommonApi.getInternalRequestHeader())); - // expect 204 because there is no session - expect(status).toBe(204); - }); - - it('extend', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .post('/internal/security/session') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [post] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .post('/internal/security/session') - .set(svlCommonApi.getInternalRequestHeader())); - // expect redirect - expect(status).toBe(302); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts deleted file mode 100644 index 79555fd0953f..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { kibanaTestUser } from '@kbn/test'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); - const samlTools = getService('samlTools'); - - describe('security/user_profiles', function () { - describe('route access', () => { - describe('internal', () => { - it('update', async () => { - const { status } = await supertestWithoutAuth - .post(`/internal/security/user_profile/_data`) - .set(svlCommonApi.getInternalRequestHeader()) - .set(await samlTools.login(kibanaTestUser.username)) - .send({ key: 'value' }); - expect(status).not.toBe(404); - }); - - it('get current', async () => { - const { status } = await supertestWithoutAuth - .get(`/internal/security/user_profile`) - .set(svlCommonApi.getInternalRequestHeader()) - .set(await samlTools.login(kibanaTestUser.username)); - expect(status).not.toBe(404); - }); - - it('bulk get', async () => { - const { status } = await supertestWithoutAuth - .get(`/internal/security/user_profile`) - .set(svlCommonApi.getInternalRequestHeader()) - .set(await samlTools.login(kibanaTestUser.username)) - .send({ uids: ['12345678'] }); - expect(status).not.toBe(404); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/users.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/users.ts deleted file mode 100644 index 2feaa8878d1e..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/users.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/users', function () { - describe('route access', () => { - // ToDo: uncomment when we disable user APIs - // describe('disabled', () => { - // it('get', async () => { - // const { body, status } = await supertest - // .get('/internal/security/users/elastic') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('get all', async () => { - // const { body, status } = await supertest - // .get('/internal/security/users') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('create/update', async () => { - // const { body, status } = await supertest - // .post(`/internal/security/users/some_testuser`) - // .set(svlCommonApi.getInternalRequestHeader()) - // .send({ username: 'some_testuser', password: 'testpassword', roles: [] }); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('delete', async () => { - // const { body, status } = await supertest - // .delete(`/internal/security/users/elastic`) - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('disable', async () => { - // const { body, status } = await supertest - // .post(`/internal/security/users/elastic/_disable`) - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('enable', async () => { - // const { body, status } = await supertest - // .post(`/internal/security/users/elastic/_enable`) - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('set password', async () => { - // const { body, status } = await supertest - // .post(`/internal/security/users/{username}/password`) - // .set(svlCommonApi.getInternalRequestHeader()) - // .send({ - // password: 'old_pw', - // newPassword: 'new_pw', - // }); - // svlCommonApi.assertApiNotFound(body, status); - // }); - // }); - - // ToDo: remove when we disable user APIs - describe('internal', () => { - it('get', async () => { - const { status } = await supertest - .get('/internal/security/users/elastic') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('get all', async () => { - const { status } = await supertest - .get('/internal/security/users') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('create/update', async () => { - const { status } = await supertest - .post(`/internal/security/users/some_testuser`) - .set(svlCommonApi.getInternalRequestHeader()) - .send({ username: 'some_testuser', password: 'testpassword', roles: [] }); - expect(status).not.toBe(404); - }); - - it('delete', async () => { - const { status } = await supertest - .delete(`/internal/security/users/elastic`) - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('disable', async () => { - const { status } = await supertest - .post(`/internal/security/users/elastic/_disable`) - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('enable', async () => { - const { status } = await supertest - .post(`/internal/security/users/elastic/_enable`) - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).not.toBe(404); - }); - - it('set password', async () => { - const { status } = await supertest - .post(`/internal/security/users/{username}/password`) - .set(svlCommonApi.getInternalRequestHeader()) - .send({ - password: 'old_pw', - newPassword: 'new_pw', - }); - expect(status).not.toBe(404); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/views.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/views.ts deleted file mode 100644 index ac1d994252c5..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/views.ts +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - describe('security/views', function () { - describe('route access', () => { - describe('disabled', () => { - // ToDo: uncomment these when we disable login routes - // it('login', async () => { - // const { body, status } = await supertest - // .get('/login') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - // it('get login state', async () => { - // const { body, status } = await supertest - // .get('/internal/security/login_state') - // .set(svlCommonApi.getInternalRequestHeader()); - // svlCommonApi.assertApiNotFound(body, status); - // }); - - it('access agreement', async () => { - const { body, status } = await supertest - .get('/security/access_agreement') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - - it('get access agreement state', async () => { - const { body, status } = await supertest - .get('/internal/security/access_agreement/state') - .set(svlCommonApi.getInternalRequestHeader()); - svlCommonApi.assertApiNotFound(body, status); - }); - }); - - describe('public', () => { - it('login', async () => { - const { status } = await supertest - .get('/login') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).toBe(302); - }); - - it('get login state', async () => { - const { status } = await supertest - .get('/internal/security/login_state') - .set(svlCommonApi.getInternalRequestHeader()); - expect(status).toBe(200); - }); - - it('capture URL', async () => { - const { status } = await supertest - .get('/internal/security/capture-url') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - - it('space selector', async () => { - const { status } = await supertest - .get('/spaces/space_selector') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - - it('enter space', async () => { - const { status } = await supertest - .get('/spaces/enter') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(302); - }); - - it('account', async () => { - const { status } = await supertest - .get('/security/account') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - - it('logged out', async () => { - const { status } = await supertest - .get('/security/logged_out') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - - it('logout', async () => { - const { status } = await supertest - .get('/logout') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - - it('overwritten session', async () => { - const { status } = await supertest - .get('/security/overwritten_session') - .set(svlCommonApi.getCommonRequestHeader()); - expect(status).toBe(200); - }); - }); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/response_headers.ts b/x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts similarity index 95% rename from x-pack/test_serverless/api_integration/test_suites/common/security/response_headers.ts rename to x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts index 56a52914cf9a..01d1c1b147aa 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/response_headers.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts @@ -6,13 +6,13 @@ */ import expect from 'expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); - describe('security/response_headers', function () { + describe('security response headers', function () { const defaultCSP = `script-src 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'; frame-ancestors 'self'`; const defaultCOOP = 'same-origin'; const defaultPermissionsPolicy = diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security_users.ts b/x-pack/test_serverless/api_integration/test_suites/common/security_users.ts new file mode 100644 index 000000000000..2c82e216505b --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security_users.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + // Test should be unskipped when the API is disabled + // https://github.com/elastic/kibana/issues/161337 + describe.skip('security/users', function () { + it('rejects request to create user', async () => { + const { body, status } = await supertest + .post(`/internal/security/users/some_testuser`) + .set(svlCommonApi.getInternalRequestHeader()) + .send({ username: 'some_testuser', password: 'testpassword', roles: [] }); + + // in a non-serverless environment this would succeed with a 200 + svlCommonApi.assertResponseStatusCode(400, status, body); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts b/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts index 78c2456f85ca..fcb3dfe84fd1 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts @@ -13,200 +13,44 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); describe('spaces', function () { - describe('route access', () => { - describe('disabled', () => { - it('#delete', async () => { - const { body, status } = await supertest - .delete('/api/spaces/space/default') - .set(svlCommonApi.getCommonRequestHeader()); - - // normally we'd get a 400 bad request if we tried to delete the default space - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#create', async () => { - const { body, status } = await supertest - .post('/api/spaces/space') - .set(svlCommonApi.getCommonRequestHeader()) - .send({ - id: 'custom', - name: 'Custom', - disabledFeatures: [], - }); - - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#update requires internal header', async () => { - const { body, status } = await supertest - .put('/api/spaces/space/default') - .set(svlCommonApi.getCommonRequestHeader()) - .send({ - id: 'default', - name: 'UPDATED!', - disabledFeatures: [], - }); - - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#copyToSpace', async () => { - const { body, status } = await supertest - .post('/api/spaces/_copy_saved_objects') - .set(svlCommonApi.getCommonRequestHeader()); - - // without a request body we would normally a 400 bad request if the endpoint was registered - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#resolveCopyToSpaceErrors', async () => { - const { body, status } = await supertest - .post('/api/spaces/_resolve_copy_saved_objects_errors') - .set(svlCommonApi.getCommonRequestHeader()); - - // without a request body we would normally a 400 bad request if the endpoint was registered - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#updateObjectsSpaces', async () => { - const { body, status } = await supertest - .post('/api/spaces/_update_objects_spaces') - .set(svlCommonApi.getCommonRequestHeader()); - - // without a request body we would normally a 400 bad request if the endpoint was registered - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#getShareableReferences', async () => { - const { body, status } = await supertest - .post('/api/spaces/_get_shareable_references') - .set(svlCommonApi.getCommonRequestHeader()); - - // without a request body we would normally a 400 bad request if the endpoint was registered - svlCommonApi.assertApiNotFound(body, status); - }); - - it('#disableLegacyUrlAliases', async () => { - const { body, status } = await supertest - .post('/api/spaces/_disable_legacy_url_aliases') - .set(svlCommonApi.getCommonRequestHeader()); - - // without a request body we would normally a 400 bad request if the endpoint was registered - svlCommonApi.assertApiNotFound(body, status); - }); + it('rejects request to create a space', async () => { + const { body, status } = await supertest + .post('/api/spaces/space') + .set(svlCommonApi.getInternalRequestHeader()) + .send({ + id: 'custom', + name: 'Custom', + disabledFeatures: [], + }); + + // in a non-serverless environment this would succeed with a 200 + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: + 'Unable to create Space, this exceeds the maximum number of spaces set by the xpack.spaces.maxSpaces setting', }); + expect(status).toBe(400); + }); - describe('internal', () => { - it('#get requires internal header', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .get('/api/spaces/space/default') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/api/spaces/space/default') - .set(svlCommonApi.getInternalRequestHeader())); - // expect success because we're using the internal header - expect(body).toEqual( - expect.objectContaining({ - id: 'default', - }) - ); - expect(status).toBe(200); - }); - - it('#getAll requires internal header', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .get('/api/spaces/space') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/api/spaces/space') - .set(svlCommonApi.getInternalRequestHeader())); - // expect success because we're using the internal header - expect(body).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - id: 'default', - }), - ]) - ); - expect(status).toBe(200); - }); - - it('#getActiveSpace requires internal header', async () => { - let body: any; - let status: number; - - ({ body, status } = await supertest - .get('/internal/spaces/_active_space') - .set(svlCommonApi.getCommonRequestHeader())); - // expect a rejection because we're not using the internal header - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: expect.stringContaining( - 'method [get] exists but is not available with the current configuration' - ), - }); - expect(status).toBe(400); - - ({ body, status } = await supertest - .get('/internal/spaces/_active_space') - .set(svlCommonApi.getInternalRequestHeader())); - // expect success because we're using the internal header - expect(body).toEqual( - expect.objectContaining({ - id: 'default', - }) - ); - expect(status).toBe(200); - }); + it('rejects request to update a space with disabledFeatures', async () => { + const { body, status } = await supertest + .put('/api/spaces/space/default') + .set(svlCommonApi.getInternalRequestHeader()) + .send({ + id: 'custom', + name: 'Custom', + disabledFeatures: ['some-feature'], + }); + + // in a non-serverless environment this would succeed with a 200 + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: + 'Unable to update Space, the disabledFeatures array must be empty when xpack.spaces.allowFeatureVisibility setting is disabled', }); + expect(status).toBe(400); }); - - // TODO: Re-enable test-suite once users can create and update spaces in the Serverless offering. - // it('rejects request to update a space with disabledFeatures', async () => { - // const { body, status } = await supertest - // .put('/api/spaces/space/default') - // .set(svlCommonApi.getInternalRequestHeader()) - // .send({ - // id: 'custom', - // name: 'Custom', - // disabledFeatures: ['some-feature'], - // }); - // - // // in a non-serverless environment this would succeed with a 200 - // expect(body).toEqual({ - // statusCode: 400, - // error: 'Bad Request', - // message: - // 'Unable to update Space, the disabledFeatures array must be empty when xpack.spaces.allowFeatureVisibility setting is disabled', - // }); - // expect(status).toBe(400); - // }); }); } diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts index a8cd016ed615..7b9d571d8c7c 100644 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts +++ b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts @@ -6,6 +6,7 @@ */ import { request } from '@kbn/security-solution-plugin/public/management/cypress/tasks/common'; +import { isLocalhost } from '@kbn/security-solution-plugin/scripts/endpoint/common/is_localhost'; import type { ServerlessRoleName } from '../../../../../shared/lib'; import { STANDARD_HTTP_HEADERS } from '../../../../../shared/lib/security/default_http_headers'; @@ -31,7 +32,7 @@ const sendApiLoginRequest = ( url: url.toString(), body: { providerType: 'basic', - providerName: 'basic', + providerName: isLocalhost(url.hostname) ? 'basic' : 'cloud-basic', currentURL: '/', params: { username, diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 40e59a747897..f65ac79e10b4 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -23,31 +23,11 @@ export default async () => { elasticsearch: esTestConfig.getUrlParts(), }; - // "Fake" SAML provider - const idpPath = resolve( - __dirname, - '../../test/security_api_integration/plugins/saml_provider/metadata.xml' - ); - const samlIdPPlugin = resolve( - __dirname, - '../../test/security_api_integration/plugins/saml_provider' - ); - return { servers, esTestCluster: { from: 'serverless', - serverArgs: [ - 'xpack.security.authc.token.enabled=true', - 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', - `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${idpPath}`, - 'xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=http://www.elastic.co/saml1', - `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://localhost:${servers.kibana.port}`, - `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://localhost:${servers.kibana.port}/logout`, - `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://localhost:${servers.kibana.port}/api/security/saml/callback`, - 'xpack.security.authc.realms.saml.cloud-saml-kibana.attributes.principal=urn:oid:0.0.7', - ], }, kbnTestServer: { @@ -58,7 +38,7 @@ export default async () => { sourceArgs: ['--no-base-path', '--env.name=development'], serverArgs: [ `--server.restrictInternalApis=true`, - `--server.port=${servers.kibana.port}`, + `--server.port=${kbnTestConfig.getPort()}`, '--status.allowAnonymous=true', // We shouldn't embed credentials into the URL since Kibana requests to Elasticsearch should // either include `kibanaServerTestUser` credentials, or credentials provided by the test @@ -87,16 +67,6 @@ export default async () => { // appenders: ['deprecation'], // }, // ])}`, - // This ensures that we register the Security SAML API endpoints. - // In the real world the SAML config is injected by control plane. - // basic: { 'basic': { order: 0 } }, - `--plugin-path=${samlIdPPlugin}`, - '--xpack.cloud.id=ftr_fake_cloud_id', - '--xpack.security.authc.selector.enabled=false', - `--xpack.security.authc.providers=${JSON.stringify({ - basic: { basic: { order: 0 } }, - saml: { 'cloud-saml-kibana': { order: 1, realm: 'cloud-saml-kibana' } }, - })}`, '--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', `--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`, ], diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index b986a6134525..eb69a3788403 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -47,6 +47,5 @@ "@kbn/core-http-common", "@kbn/data-views-plugin", "@kbn/core-saved-objects-server", - "@kbn/security-api-integration-helpers", ] } From 189a0d7ad4b96c7551aacd788441637823621ef7 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 24 Aug 2023 19:51:35 +0100 Subject: [PATCH 101/182] chore(NA): docker login on serverless osquery --- .../steps/functional/security_serverless_osquery.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/functional/security_serverless_osquery.sh b/.buildkite/scripts/steps/functional/security_serverless_osquery.sh index 60312fcaf681..d631424552fa 100755 --- a/.buildkite/scripts/steps/functional/security_serverless_osquery.sh +++ b/.buildkite/scripts/steps/functional/security_serverless_osquery.sh @@ -2,10 +2,13 @@ set -euo pipefail -source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/functional/common.sh source .buildkite/scripts/steps/functional/common_cypress.sh -.buildkite/scripts/bootstrap.sh +# TODO: remove the line below to use build artifacts for tests. +# in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location +# by exporting a var like: +# export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} node scripts/build_kibana_platform_plugins.js export JOB=kibana-osquery-cypress-serverless From a8b0748b202d038318747383c9141a8d5e1355ed Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 24 Aug 2023 18:28:40 -0600 Subject: [PATCH 102/182] Support secrets.json. Fix deprecation warnings for SSL --- .../kbn-es/src/ess_resources/secrets.json | 10 +++++++++ packages/kbn-es/src/paths.ts | 2 ++ packages/kbn-es/src/utils/docker.ts | 22 +++++++++++++------ x-pack/test_serverless/shared/config.base.ts | 14 ++++++------ 4 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 packages/kbn-es/src/ess_resources/secrets.json diff --git a/packages/kbn-es/src/ess_resources/secrets.json b/packages/kbn-es/src/ess_resources/secrets.json new file mode 100644 index 000000000000..c19d78f7bd53 --- /dev/null +++ b/packages/kbn-es/src/ess_resources/secrets.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "version": "1", + "compatibility": "8.11.0" + }, + "string_secrets": { + "xpack.security.http.ssl.keystore.secure_password": "storepass", + "xpack.security.transport.ssl.keystore.secure_password": "storepass" + } +} \ No newline at end of file diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 26cee48b11b3..63effda7123c 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -33,6 +33,8 @@ export const ESS_USERS_ROLES_PATH = resolve(__dirname, './ess_resources/users_ro export const ESS_ROLES_PATH = resolve(__dirname, './ess_resources/roles.yml'); export const ESS_ROLE_MAPPING_PATH = resolve(__dirname, './ess_resources/role_mapping.yml'); +export const ESS_SECRETS_PATH = resolve(__dirname, './ess_resources/secrets.json'); + export const ESS_RESOURCES_PATHS = [ ESS_OPERATOR_USERS_PATH, ESS_ROLE_MAPPING_PATH, diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 588ab7ec9bd4..74d32173b5fb 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -17,7 +17,7 @@ import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_RESOURCES_PATHS } from '../paths'; +import { ESS_RESOURCES_PATHS, ESS_SECRETS_PATH } from '../paths'; interface BaseOptions { tag?: string; @@ -138,21 +138,23 @@ const DEFAULT_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.http.ssl.keystore.path', `${ESS_CONFIG_PATH}certs/elasticsearch.p12`], - ['xpack.security.http.ssl.keystore.password', ES_P12_PASSWORD], - ['xpack.security.http.ssl.verification_mode', 'certificate'], ['xpack.security.transport.ssl.enabled', 'true'], ['xpack.security.transport.ssl.keystore.path', `${ESS_CONFIG_PATH}certs/elasticsearch.p12`], - ['xpack.security.transport.ssl.keystore.password', ES_P12_PASSWORD], - ['xpack.security.transport.ssl.verification_mode', 'certificate'], ['xpack.security.operator_privileges.enabled', 'true'], ]; +const DOCKER_SSL_ESARGS: Array<[string, string]> = [ + ['xpack.security.http.ssl.keystore.password', ES_P12_PASSWORD], + + ['xpack.security.transport.ssl.keystore.password', ES_P12_PASSWORD], +]; + const SERVERLESS_NODES: Array> = [ { name: 'es01', @@ -416,7 +418,13 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - const baseCmd = ['--volume', `${basePath}:/objectstore:z`]; + const baseCmd = [ + '--volume', + `${basePath}:/objectstore:z`, + + '--volume', + `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, + ]; if (ssl) { const essResources = ESS_RESOURCES_PATHS.reduce((acc, path) => { @@ -554,7 +562,7 @@ export function resolveDockerCmd(options: DockerOptions, image: string = DOCKER_ } return DOCKER_BASE_CMD.concat( - resolveEsArgs(DEFAULT_DOCKER_ESARGS, options), + resolveEsArgs(DEFAULT_DOCKER_ESARGS.concat(options.ssl ? DOCKER_SSL_ESARGS : []), options), resolvePort(options), options.ssl ? getESp12Volume() : [], image diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 7b55a3e7627b..2acd09dd5350 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -64,13 +64,13 @@ export default async () => { * ESS emits deprecation warnings for ssl.keystore.password. * Need to mount a secure keystore into the images because ES_NOPASSWORD_P12_PATH doesn't work. */ - // `--logging.loggers=${JSON.stringify([ - // { - // name: 'elasticsearch.deprecation', - // level: 'all', - // appenders: ['deprecation'], - // }, - // ])}`, + `--logging.loggers=${JSON.stringify([ + { + name: 'elasticsearch.deprecation', + level: 'all', + appenders: ['deprecation'], + }, + ])}`, '--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', `--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`, ], From cb8e6bf95382b0e62b9ed7379ab896b3229fa5c8 Mon Sep 17 00:00:00 2001 From: Brad White Date: Thu, 24 Aug 2023 18:30:31 -0600 Subject: [PATCH 103/182] Remove comment about deprecation --- x-pack/test_serverless/shared/config.base.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 2acd09dd5350..c2d4fd700263 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -60,10 +60,6 @@ export default async () => { type: 'json', }, })}`, - /** - * ESS emits deprecation warnings for ssl.keystore.password. - * Need to mount a secure keystore into the images because ES_NOPASSWORD_P12_PATH doesn't work. - */ `--logging.loggers=${JSON.stringify([ { name: 'elasticsearch.deprecation', From 23f4c8a79036e39060116d6de851c896d518307b Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 25 Aug 2023 13:47:39 +0100 Subject: [PATCH 104/182] chore(NA): add service token when running kibana dev mode in serverless --- src/cli/serve/serve.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 5027d0484fd1..3998e07417d1 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -15,6 +15,7 @@ import { isKibanaDistributable } from '@kbn/repo-info'; import { readKeystore } from '../keystore/read_keystore'; import { compileConfigStack } from './compile_config_stack'; import { getConfigFromFiles } from '@kbn/config'; +import { kibanaServiceAccount } from '@kbn/test'; const DEV_MODE_PATH = '@kbn/cli-dev-mode'; const DEV_MODE_SUPPORTED = canRequire(DEV_MODE_PATH); @@ -68,6 +69,10 @@ export function applyConfigOverrides(rawConfig, opts, extraCliOptions) { delete extraCliOptions.env; if (opts.dev) { + if (opts.serverless) { + set('elasticsearch.serviceAccountToken', kibanaServiceAccount.token); + } + if (!has('elasticsearch.serviceAccountToken') && opts.devCredentials !== false) { if (!has('elasticsearch.username')) { set('elasticsearch.username', 'kibana_system'); From 4f61f5071c0adb18ab9a8c410f8b17a6398a46fe Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:53:53 +0000 Subject: [PATCH 105/182] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/cli/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/tsconfig.json b/src/cli/tsconfig.json index ebbbc19f75c7..3f14fafce07b 100644 --- a/src/cli/tsconfig.json +++ b/src/cli/tsconfig.json @@ -17,6 +17,7 @@ "@kbn/config", "@kbn/dev-utils", "@kbn/apm-config-loader", + "@kbn/test", ], "exclude": [ "target/**/*", From c5181f9ff3b5928d4811ed4c0ccc946e66e6f817 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 25 Aug 2023 14:05:27 +0100 Subject: [PATCH 106/182] chore(NA): only load secrets.json into the volume when ssl --- packages/kbn-es/src/utils/docker.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 74d32173b5fb..596952293a5a 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -420,10 +420,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles const baseCmd = [ '--volume', - `${basePath}:/objectstore:z`, - - '--volume', - `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, + `${basePath}:/objectstore:z` ]; if (ssl) { @@ -435,7 +432,14 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return acc; }, []); - return baseCmd.concat(getESp12Volume(), essResources); + return baseCmd.concat( + getESp12Volume(), + essResources, + [ + '--volume', + `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, + ] + ); } return baseCmd; From aba550d6f7c64e14c38cee0e7d33cd4806880fbe Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 25 Aug 2023 13:11:45 +0000 Subject: [PATCH 107/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- packages/kbn-es/src/utils/docker.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 596952293a5a..1eff21fb8f0e 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -418,10 +418,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles log.indent(-4); - const baseCmd = [ - '--volume', - `${basePath}:/objectstore:z` - ]; + const baseCmd = ['--volume', `${basePath}:/objectstore:z`]; if (ssl) { const essResources = ESS_RESOURCES_PATHS.reduce((acc, path) => { @@ -432,14 +429,10 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles return acc; }, []); - return baseCmd.concat( - getESp12Volume(), - essResources, - [ - '--volume', - `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, - ] - ); + return baseCmd.concat(getESp12Volume(), essResources, [ + '--volume', + `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, + ]); } return baseCmd; From 6f700c6cacc393d9f89b5509ed63e7043da2e70c Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Fri, 25 Aug 2023 15:49:00 +0200 Subject: [PATCH 108/182] skip tests depended on alerts --- .../data_sources/create_runtime_field.cy.ts | 64 ++++----- .../e2e/detection_alerts/alert_tags.cy.ts | 2 +- .../e2e/detection_alerts/enrichments.cy.ts | 2 +- .../ransomware_detection.cy.ts | 70 +++++----- .../ransomware_prevention.cy.ts | 76 +++++------ .../explore/cases/attach_alert_to_case.cy.ts | 6 +- .../alerts/alert_table_action_column.cy.ts | 48 +++---- .../alerts/alerts_details.cy.ts | 2 +- .../alerts/investigate_in_timeline.cy.ts | 122 +++++++++--------- .../e2e/investigations/alerts/resolver.cy.ts | 50 +++---- .../timelines/bulk_add_to_timeline.cy.ts | 2 +- 11 files changed, 234 insertions(+), 210 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts index 60f75e3a89a4..b9a514b8c221 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts @@ -25,33 +25,37 @@ import { GET_TIMELINE_HEADER } from '../../screens/timeline'; const alertRunTimeField = 'field.name.alert.page'; const timelineRuntimeField = 'field.name.timeline'; -describe('Create DataView runtime field', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - deleteRuntimeField('security-solution-default', alertRunTimeField); - deleteRuntimeField('security-solution-default', timelineRuntimeField); - }); - - beforeEach(() => { - login(); - }); - - it('adds field to alert table', () => { - visit(ALERTS_URL); - createRule(getNewRule()); - refreshPage(); - waitForAlertsToPopulate(); - openAlertsFieldBrowser(); - createField(alertRunTimeField); - cy.get(GET_DATA_GRID_HEADER(alertRunTimeField)).should('exist'); - }); - - it('adds field to timeline', () => { - visit(HOSTS_URL); - openTimelineUsingToggle(); - populateTimeline(); - openTimelineFieldsBrowser(); - - createField(timelineRuntimeField); - cy.get(GET_TIMELINE_HEADER(timelineRuntimeField)).should('exist'); - }); -}); +describe( + 'Create DataView runtime field', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + deleteRuntimeField('security-solution-default', alertRunTimeField); + deleteRuntimeField('security-solution-default', timelineRuntimeField); + }); + + beforeEach(() => { + login(); + }); + + it('adds field to alert table', () => { + visit(ALERTS_URL); + createRule(getNewRule()); + refreshPage(); + waitForAlertsToPopulate(); + openAlertsFieldBrowser(); + createField(alertRunTimeField); + cy.get(GET_DATA_GRID_HEADER(alertRunTimeField)).should('exist'); + }); + + it('adds field to timeline', () => { + visit(HOSTS_URL); + openTimelineUsingToggle(); + populateTimeline(); + openTimelineFieldsBrowser(); + + createField(timelineRuntimeField); + cy.get(GET_TIMELINE_HEADER(timelineRuntimeField)).should('exist'); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alert_tags.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alert_tags.cy.ts index 39758e5a097c..0a97eb7de122 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alert_tags.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alert_tags.cy.ts @@ -24,7 +24,7 @@ import { UNSELECTED_ALERT_TAG, } from '../../screens/alerts'; -describe('Alert tagging', { tags: ['@ess', '@serverless'] }, () => { +describe('Alert tagging', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); cy.task('esArchiverResetKibana'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/enrichments.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/enrichments.cy.ts index 3cf0665795cf..47c5f0873b56 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/enrichments.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/enrichments.cy.ts @@ -30,7 +30,7 @@ import { login, visit } from '../../tasks/login'; import { ALERTS_URL } from '../../urls/navigation'; -describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => { +describe('Enrichment', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); cy.task('esArchiverLoad', 'risk_users'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts index 93e0ce8ba8f4..f55265ac1547 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts @@ -14,49 +14,53 @@ import { TIMELINE_QUERY, TIMELINE_VIEW_IN_ANALYZER } from '../../screens/timelin import { selectAlertsHistogram } from '../../tasks/alerts'; import { createTimeline } from '../../tasks/timelines'; -describe('Ransomware Detection Alerts', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cy.task('esArchiverLoad', 'ransomware_detection'); - }); - - describe('Ransomware display in Alerts Section', () => { - beforeEach(() => { - login(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); +describe( + 'Ransomware Detection Alerts', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + cy.task('esArchiverLoad', 'ransomware_detection'); }); - describe('Alerts table', () => { - it('shows Ransomware Alerts', () => { - cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Detection Alert'); + describe('Ransomware display in Alerts Section', () => { + beforeEach(() => { + login(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); }); - }); - describe('Trend Chart', () => { - beforeEach(() => { - selectAlertsHistogram(); + describe('Alerts table', () => { + it('shows Ransomware Alerts', () => { + cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Detection Alert'); + }); }); - it('shows Ransomware Detection Alert in the trend chart', () => { - cy.get(ALERTS_HISTOGRAM_SERIES).should('have.text', 'Ransomware Detection Alert'); + describe('Trend Chart', () => { + beforeEach(() => { + selectAlertsHistogram(); + }); + + it('shows Ransomware Detection Alert in the trend chart', () => { + cy.get(ALERTS_HISTOGRAM_SERIES).should('have.text', 'Ransomware Detection Alert'); + }); }); }); - }); - describe('Ransomware in Timelines', () => { - before(() => { - login(); - visit(TIMELINES_URL); - createTimeline(); - }); + describe('Ransomware in Timelines', () => { + before(() => { + login(); + visit(TIMELINES_URL); + createTimeline(); + }); - it('Renders ransomware entries in timelines table', () => { - cy.get(TIMELINE_QUERY).type('event.code: "ransomware"{enter}'); + it('Renders ransomware entries in timelines table', () => { + cy.get(TIMELINE_QUERY).type('event.code: "ransomware"{enter}'); - // Wait for grid to load, it should have an analyzer icon - cy.get(TIMELINE_VIEW_IN_ANALYZER).should('exist'); + // Wait for grid to load, it should have an analyzer icon + cy.get(TIMELINE_VIEW_IN_ANALYZER).should('exist'); - cy.get(MESSAGE).should('have.text', 'Ransomware Detection Alert'); + cy.get(MESSAGE).should('have.text', 'Ransomware Detection Alert'); + }); }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts index 4f6379b3a6bd..9896545a1ec5 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts @@ -14,54 +14,58 @@ import { TIMELINE_QUERY, TIMELINE_VIEW_IN_ANALYZER } from '../../screens/timelin import { selectAlertsHistogram } from '../../tasks/alerts'; import { createTimeline } from '../../tasks/timelines'; -describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cy.task('esArchiverLoad', 'ransomware_prevention'); - }); - - after(() => { - cy.task('esArchiverUnload', 'ransomware_prevention'); - }); - - describe('Ransomware display in Alerts Section', () => { - beforeEach(() => { - login(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); +describe( + 'Ransomware Prevention Alerts', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + cy.task('esArchiverLoad', 'ransomware_prevention'); }); - describe('Alerts table', () => { - it('shows Ransomware Alerts', () => { - cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Prevention Alert'); - }); + after(() => { + cy.task('esArchiverUnload', 'ransomware_prevention'); }); - describe('Trend Chart', () => { + describe('Ransomware display in Alerts Section', () => { beforeEach(() => { - selectAlertsHistogram(); + login(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); }); - it('shows Ransomware Prevention Alert in the trend chart', () => { - cy.get(ALERTS_HISTOGRAM_SERIES).should('have.text', 'Ransomware Prevention Alert'); + describe('Alerts table', () => { + it('shows Ransomware Alerts', () => { + cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Prevention Alert'); + }); }); - }); - }); - describe('Ransomware in Timelines', () => { - beforeEach(() => { - login(); - visit(TIMELINES_URL); + describe('Trend Chart', () => { + beforeEach(() => { + selectAlertsHistogram(); + }); - createTimeline(); + it('shows Ransomware Prevention Alert in the trend chart', () => { + cy.get(ALERTS_HISTOGRAM_SERIES).should('have.text', 'Ransomware Prevention Alert'); + }); + }); }); - it('Renders ransomware entries in timelines table', () => { - cy.get(TIMELINE_QUERY).type('event.code: "ransomware"{enter}'); + describe('Ransomware in Timelines', () => { + beforeEach(() => { + login(); + visit(TIMELINES_URL); + + createTimeline(); + }); - // Wait for grid to load, it should have an analyzer icon - cy.get(TIMELINE_VIEW_IN_ANALYZER).should('exist'); + it('Renders ransomware entries in timelines table', () => { + cy.get(TIMELINE_QUERY).type('event.code: "ransomware"{enter}'); - cy.get(MESSAGE).should('have.text', 'Ransomware Prevention Alert'); + // Wait for grid to load, it should have an analyzer icon + cy.get(TIMELINE_VIEW_IN_ANALYZER).should('exist'); + + cy.get(MESSAGE).should('have.text', 'Ransomware Prevention Alert'); + }); }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/attach_alert_to_case.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/attach_alert_to_case.cy.ts index 4762c88c551c..bb84c0a8004e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/attach_alert_to_case.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/attach_alert_to_case.cy.ts @@ -24,7 +24,7 @@ const loadDetectionsPage = (role: ROLES) => { waitForAlertsToPopulate(); }; -describe('Alerts timeline', { tags: '@ess' }, () => { +describe('Alerts timeline', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { // First we login as a privileged user to create alerts. cleanKibana(); @@ -34,7 +34,7 @@ describe('Alerts timeline', { tags: '@ess' }, () => { waitForAlertsToPopulate(); }); - context('Privileges: read only', { tags: '@ess' }, () => { + context('Privileges: read only', () => { beforeEach(() => { loadDetectionsPage(ROLES.reader); }); @@ -52,7 +52,7 @@ describe('Alerts timeline', { tags: '@ess' }, () => { }); }); - context('Privileges: can crud', { tags: '@ess' }, () => { + context('Privileges: can crud', () => { beforeEach(() => { loadDetectionsPage(ROLES.platform_engineer); cy.get(LOADING_INDICATOR).should('not.exist'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alert_table_action_column.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alert_table_action_column.cy.ts index a8acec0019db..9bffb513cdba 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alert_table_action_column.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alert_table_action_column.cy.ts @@ -15,29 +15,33 @@ import { waitForAlertsToPopulate } from '../../../tasks/create_new_rule'; import { login, visit } from '../../../tasks/login'; import { ALERTS_URL } from '../../../urls/navigation'; -describe('Alerts Table Action column', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - cy.task('esArchiverLoad', 'process_ancestry'); - }); +describe( + 'Alerts Table Action column', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + cleanKibana(); + cy.task('esArchiverLoad', 'process_ancestry'); + }); - beforeEach(() => { - login(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); - }); + beforeEach(() => { + login(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); + }); - after(() => { - cy.task('esArchiverUnload', 'process_ancestry'); - }); + after(() => { + cy.task('esArchiverUnload', 'process_ancestry'); + }); - it('should have session viewer button visible & open session viewer on click', () => { - openSessionViewerFromAlertTable(); - cy.get(OVERLAY_CONTAINER).should('be.visible'); - }); + it('should have session viewer button visible & open session viewer on click', () => { + openSessionViewerFromAlertTable(); + cy.get(OVERLAY_CONTAINER).should('be.visible'); + }); - it('should have analyzer button visible & open analyzer on click', () => { - openAnalyzerForFirstAlertInTimeline(); - cy.get(OVERLAY_CONTAINER).should('be.visible'); - }); -}); + it('should have analyzer button visible & open analyzer on click', () => { + openAnalyzerForFirstAlertInTimeline(); + cy.get(OVERLAY_CONTAINER).should('be.visible'); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts index d4ecaf16df97..f74dd50eddbf 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/alerts_details.cy.ts @@ -35,7 +35,7 @@ import { ALERT_SUMMARY_SEVERITY_DONUT_CHART } from '../../../screens/alerts'; import { getLocalstorageEntryAsObject } from '../../../helpers/common'; import { goToRuleDetails } from '../../../tasks/alerts_detection_rules'; -describe('Alert details flyout', { tags: ['@ess', '@serverless'] }, () => { +describe('Alert details flyout', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { describe('Basic functions', () => { before(() => { cleanKibana(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/investigate_in_timeline.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/investigate_in_timeline.cy.ts index 22a80ba6fda9..59df434ff029 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/investigate_in_timeline.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/investigate_in_timeline.cy.ts @@ -27,76 +27,80 @@ import { } from '../../../screens/alerts_details'; import { verifyInsightCount } from '../../../tasks/alerts_details'; -describe('Investigate in timeline', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - createRule(getNewRule()); - }); - - describe('From alerts table', () => { - beforeEach(() => { - login(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); +describe( + 'Investigate in timeline', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + cleanKibana(); + createRule(getNewRule()); }); - it('should open new timeline from alerts table', () => { - investigateFirstAlertInTimeline(); - cy.get(PROVIDER_BADGE) - .first() - .invoke('text') - .then((eventId) => { - cy.get(PROVIDER_BADGE).filter(':visible').should('have.text', eventId); - }); - }); - }); + describe('From alerts table', () => { + beforeEach(() => { + login(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); + }); - describe('From alerts details flyout', () => { - beforeEach(() => { - login(); - disableExpandableFlyout(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); - expandFirstAlert(); + it('should open new timeline from alerts table', () => { + investigateFirstAlertInTimeline(); + cy.get(PROVIDER_BADGE) + .first() + .invoke('text') + .then((eventId) => { + cy.get(PROVIDER_BADGE).filter(':visible').should('have.text', eventId); + }); + }); }); - it('should open a new timeline from a prevalence field', () => { - // Only one alert matches the exact process args in this case - const alertCount = 1; + describe('From alerts details flyout', () => { + beforeEach(() => { + login(); + disableExpandableFlyout(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); + expandFirstAlert(); + }); + + it('should open a new timeline from a prevalence field', () => { + // Only one alert matches the exact process args in this case + const alertCount = 1; - // Click on the last button that lets us investigate in timeline. - // We expect this to be the `process.args` row. - cy.get(ALERT_FLYOUT) - .find(SUMMARY_VIEW_INVESTIGATE_IN_TIMELINE_BUTTON) - .last() - .should('have.text', alertCount) - .click(); + // Click on the last button that lets us investigate in timeline. + // We expect this to be the `process.args` row. + cy.get(ALERT_FLYOUT) + .find(SUMMARY_VIEW_INVESTIGATE_IN_TIMELINE_BUTTON) + .last() + .should('have.text', alertCount) + .click(); - // Make sure a new timeline is created and opened - cy.get(TIMELINE_TITLE).should('have.text', 'Untitled timeline'); + // Make sure a new timeline is created and opened + cy.get(TIMELINE_TITLE).should('have.text', 'Untitled timeline'); - // The alert count in this timeline should match the count shown on the alert flyout - cy.get(QUERY_TAB_BUTTON).should('contain.text', alertCount); + // The alert count in this timeline should match the count shown on the alert flyout + cy.get(QUERY_TAB_BUTTON).should('contain.text', alertCount); - // The correct filter is applied to the timeline query - cy.get(FILTER_BADGE).should( - 'have.text', - ' {"bool":{"must":[{"term":{"process.args":"-zsh"}},{"term":{"process.args":"unique"}}]}}' - ); - }); + // The correct filter is applied to the timeline query + cy.get(FILTER_BADGE).should( + 'have.text', + ' {"bool":{"must":[{"term":{"process.args":"-zsh"}},{"term":{"process.args":"unique"}}]}}' + ); + }); - it('should open a new timeline from an insights module', () => { - verifyInsightCount({ - tableSelector: INSIGHTS_RELATED_ALERTS_BY_SESSION, - investigateSelector: INSIGHTS_INVESTIGATE_IN_TIMELINE_BUTTON, + it('should open a new timeline from an insights module', () => { + verifyInsightCount({ + tableSelector: INSIGHTS_RELATED_ALERTS_BY_SESSION, + investigateSelector: INSIGHTS_INVESTIGATE_IN_TIMELINE_BUTTON, + }); }); - }); - it('should open a new timeline with alert ids from the process ancestry', () => { - verifyInsightCount({ - tableSelector: INSIGHTS_RELATED_ALERTS_BY_ANCESTRY, - investigateSelector: INSIGHTS_INVESTIGATE_ANCESTRY_ALERTS_IN_TIMELINE_BUTTON, + it('should open a new timeline with alert ids from the process ancestry', () => { + verifyInsightCount({ + tableSelector: INSIGHTS_RELATED_ALERTS_BY_ANCESTRY, + investigateSelector: INSIGHTS_INVESTIGATE_ANCESTRY_ALERTS_IN_TIMELINE_BUTTON, + }); }); }); - }); -}); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/resolver.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/resolver.cy.ts index 722e19670685..0132acaad598 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/resolver.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/resolver.cy.ts @@ -17,29 +17,33 @@ import { waitForAlertsToPopulate } from '../../../tasks/create_new_rule'; import { login, visit } from '../../../tasks/login'; import { ALERTS_URL } from '../../../urls/navigation'; -describe('Analyze events view for alerts', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - createRule(getNewRule()); - }); +describe( + 'Analyze events view for alerts', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + cleanKibana(); + createRule(getNewRule()); + }); - beforeEach(() => { - login(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); - }); + beforeEach(() => { + login(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); + }); - it('should render when button is clicked', () => { - openAnalyzerForFirstAlertInTimeline(); - cy.get(ANALYZER_NODE).first().should('be.visible'); - }); + it('should render when button is clicked', () => { + openAnalyzerForFirstAlertInTimeline(); + cy.get(ANALYZER_NODE).first().should('be.visible'); + }); - it('should display a toast indicating the date range of found events when a time range has 0 events in it', () => { - const dateContainingZeroEvents = 'Jul 27, 2022 @ 00:00:00.000'; - setStartDate(dateContainingZeroEvents); - waitForAlertsToPopulate(); - openAnalyzerForFirstAlertInTimeline(); - cy.get(TOASTER).should('be.visible'); - cy.get(ANALYZER_NODE).first().should('be.visible'); - }); -}); + it('should display a toast indicating the date range of found events when a time range has 0 events in it', () => { + const dateContainingZeroEvents = 'Jul 27, 2022 @ 00:00:00.000'; + setStartDate(dateContainingZeroEvents); + waitForAlertsToPopulate(); + openAnalyzerForFirstAlertInTimeline(); + cy.get(TOASTER).should('be.visible'); + cy.get(ANALYZER_NODE).first().should('be.visible'); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/bulk_add_to_timeline.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/bulk_add_to_timeline.cy.ts index 3470c21aa0ed..4a451bf8f58c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/bulk_add_to_timeline.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/bulk_add_to_timeline.cy.ts @@ -33,7 +33,7 @@ describe('Bulk Investigate in Timeline', { tags: ['@ess', '@serverless'] }, () = cy.task('esArchiverUnload', 'bulk_process'); }); - context('Alerts', () => { + context('Alerts', { tags: ['@brokenInServerless'] }, () => { before(() => { createRule(getNewRule()); }); From bc7b1ab1696eaea631c7920e081845ae66bc9264 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 25 Aug 2023 08:22:05 -0600 Subject: [PATCH 109/182] Revert "Revert "Prepare the Security domain HTTP APIs for Serverless (#162087)"" This reverts commit 35e777e29e7e18611fcea4aae1f186e57967860e. --- config/serverless.yml | 3 + .../encrypted_saved_objects/server/plugin.ts | 30 ++- x-pack/plugins/security/server/plugin.ts | 1 + .../server/routes/authentication/common.ts | 46 ++-- .../server/routes/authentication/saml.ts | 7 +- .../server/routes/authorization/index.ts | 12 +- .../plugins/security/server/routes/index.ts | 23 +- .../server/routes/session_management/index.ts | 9 +- .../server/routes/views/index.test.ts | 55 ++++- .../security/server/routes/views/index.ts | 20 +- x-pack/plugins/spaces/server/plugin.ts | 25 +- .../routes/api/external/copy_to_space.test.ts | 2 +- .../routes/api/external/copy_to_space.ts | 6 +- .../server/routes/api/external/delete.test.ts | 2 +- .../server/routes/api/external/delete.ts | 4 +- .../disable_legacy_url_aliases.test.ts | 2 +- .../external/disable_legacy_url_aliases.ts | 4 +- .../server/routes/api/external/get.test.ts | 2 +- .../spaces/server/routes/api/external/get.ts | 4 +- .../routes/api/external/get_all.test.ts | 2 +- .../server/routes/api/external/get_all.ts | 4 +- .../external/get_shareable_references.test.ts | 2 +- .../api/external/get_shareable_references.ts | 4 +- .../server/routes/api/external/index.ts | 26 +- .../server/routes/api/external/post.test.ts | 2 +- .../spaces/server/routes/api/external/post.ts | 4 +- .../server/routes/api/external/put.test.ts | 2 +- .../spaces/server/routes/api/external/put.ts | 4 +- .../external/update_objects_spaces.test.ts | 2 +- .../api/external/update_objects_spaces.ts | 4 +- .../api/internal/get_active_space.test.ts | 2 +- .../routes/api/internal/get_active_space.ts | 4 +- .../server/routes/api/internal/index.ts | 2 +- .../spaces_client/spaces_client.test.ts | 18 +- x-pack/plugins/spaces/tsconfig.json | 1 + .../api_integration/services/index.ts | 2 + .../api_integration/services/saml_tools.ts | 42 ++++ .../services/svl_common_api.ts | 9 + .../common/encrypted_saved_objects.ts | 26 ++ .../test_suites/common/index.ts | 14 +- .../test_suites/common/security/anonymous.ts | 33 +++ .../test_suites/common/security/api_keys.ts | 212 ++++++++++++++++ .../common/security/authentication.ts | 201 ++++++++++++++++ .../common/security/authorization.ts | 110 +++++++++ .../test_suites/common/security/misc.ts | 58 +++++ .../response_headers.ts} | 4 +- .../common/security/role_mappings.ts | 55 +++++ .../test_suites/common/security/sessions.ts | 78 ++++++ .../common/security/user_profiles.ts | 48 ++++ .../test_suites/common/security/users.ts | 132 ++++++++++ .../test_suites/common/security/views.ts | 114 +++++++++ .../test_suites/common/security_users.ts | 27 --- .../test_suites/common/spaces.ts | 226 +++++++++++++++--- .../security/cypress/tasks/login.ts | 3 +- x-pack/test_serverless/shared/config.base.ts | 31 ++- x-pack/test_serverless/tsconfig.json | 1 + 56 files changed, 1580 insertions(+), 186 deletions(-) create mode 100644 x-pack/test_serverless/api_integration/services/saml_tools.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts rename x-pack/test_serverless/api_integration/test_suites/common/{security_response_headers.ts => security/response_headers.ts} (95%) create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/users.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/views.ts delete mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security_users.ts diff --git a/config/serverless.yml b/config/serverless.yml index 3b0587d45cae..e18049e8517c 100644 --- a/config/serverless.yml +++ b/config/serverless.yml @@ -50,6 +50,9 @@ xpack.cloud_integrations.data_migration.enabled: false data.search.sessions.enabled: false advanced_settings.enabled: false +# Disable the browser-side functionality that depends on SecurityCheckupGetStateRoutes +xpack.security.showInsecureClusterWarning: false + # Disable UI of security management plugins xpack.security.ui.userManagementEnabled: false xpack.security.ui.roleManagementEnabled: false diff --git a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts index 92a663fab64d..7b7fed43f518 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts @@ -95,19 +95,23 @@ export class EncryptedSavedObjectsPlugin getStartServices: core.getStartServices, }); - defineRoutes({ - router: core.http.createRouter(), - logger: this.initializerContext.logger.get('routes'), - encryptionKeyRotationService: Object.freeze( - new EncryptionKeyRotationService({ - logger: this.logger.get('key-rotation-service'), - service, - getStartServices: core.getStartServices, - security: deps.security, - }) - ), - config, - }); + // In the serverless environment, the encryption keys for saved objects is managed internally and never + // exposed to users and administrators, eliminating the need for any public Encrypted Saved Objects HTTP APIs + if (this.initializerContext.env.packageInfo.buildFlavor !== 'serverless') { + defineRoutes({ + router: core.http.createRouter(), + logger: this.initializerContext.logger.get('routes'), + encryptionKeyRotationService: Object.freeze( + new EncryptionKeyRotationService({ + logger: this.logger.get('key-rotation-service'), + service, + getStartServices: core.getStartServices, + security: deps.security, + }) + ), + config, + }); + } return { canEncrypt, diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index 4f36c0bf508d..d196dd9f4113 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -359,6 +359,7 @@ export class SecurityPlugin getAnonymousAccessService: this.getAnonymousAccess, getUserProfileService: this.getUserProfileService, analyticsService: this.analyticsService.setup({ analytics: core.analytics }), + buildFlavor: this.initializerContext.env.packageInfo.buildFlavor, }); return Object.freeze({ diff --git a/x-pack/plugins/security/server/routes/authentication/common.ts b/x-pack/plugins/security/server/routes/authentication/common.ts index 4eeeed299809..f359c320151e 100644 --- a/x-pack/plugins/security/server/routes/authentication/common.ts +++ b/x-pack/plugins/security/server/routes/authentication/common.ts @@ -32,9 +32,14 @@ export function defineCommonRoutes({ basePath, license, logger, + buildFlavor, }: RouteDefinitionParams) { // Generate two identical routes with new and deprecated URL and issue a warning if route with deprecated URL is ever used. - for (const path of ['/api/security/logout', '/api/security/v1/logout']) { + // For a serverless build, do not register deprecated versioned routes + for (const path of [ + '/api/security/logout', + ...(buildFlavor !== 'serverless' ? ['/api/security/v1/logout'] : []), + ]) { router.get( { path, @@ -79,7 +84,11 @@ export function defineCommonRoutes({ } // Generate two identical routes with new and deprecated URL and issue a warning if route with deprecated URL is ever used. - for (const path of ['/internal/security/me', '/api/security/v1/me']) { + // For a serverless build, do not register deprecated versioned routes + for (const path of [ + '/internal/security/me', + ...(buildFlavor !== 'serverless' ? ['/api/security/v1/me'] : []), + ]) { router.get( { path, validate: false }, createLicensedRouteHandler((context, request, response) => { @@ -123,6 +132,8 @@ export function defineCommonRoutes({ return undefined; } + // Register the login route for serverless for the time being. Note: This route will move into the buildFlavor !== 'serverless' block below. See next line. + // ToDo: In the serverless environment, we do not support API login - the only valid authentication methodology (or maybe just method or mechanism?) is SAML router.post( { path: '/internal/security/login', @@ -169,20 +180,23 @@ export function defineCommonRoutes({ }) ); - router.post( - { path: '/internal/security/access_agreement/acknowledge', validate: false }, - createLicensedRouteHandler(async (context, request, response) => { - // If license doesn't allow access agreement we shouldn't handle request. - if (!license.getFeatures().allowAccessAgreement) { - logger.warn(`Attempted to acknowledge access agreement when license doesn't allow it.`); - return response.forbidden({ - body: { message: `Current license doesn't support access agreement.` }, - }); - } + if (buildFlavor !== 'serverless') { + // In the serverless offering, the access agreement functionality isn't available. + router.post( + { path: '/internal/security/access_agreement/acknowledge', validate: false }, + createLicensedRouteHandler(async (context, request, response) => { + // If license doesn't allow access agreement we shouldn't handle request. + if (!license.getFeatures().allowAccessAgreement) { + logger.warn(`Attempted to acknowledge access agreement when license doesn't allow it.`); + return response.forbidden({ + body: { message: `Current license doesn't support access agreement.` }, + }); + } - await getAuthenticationService().acknowledgeAccessAgreement(request); + await getAuthenticationService().acknowledgeAccessAgreement(request); - return response.noContent(); - }) - ); + return response.noContent(); + }) + ); + } } diff --git a/x-pack/plugins/security/server/routes/authentication/saml.ts b/x-pack/plugins/security/server/routes/authentication/saml.ts index 350f3527f331..ddc31fbc88b8 100644 --- a/x-pack/plugins/security/server/routes/authentication/saml.ts +++ b/x-pack/plugins/security/server/routes/authentication/saml.ts @@ -19,9 +19,14 @@ export function defineSAMLRoutes({ getAuthenticationService, basePath, logger, + buildFlavor, }: RouteDefinitionParams) { // Generate two identical routes with new and deprecated URL and issue a warning if route with deprecated URL is ever used. - for (const path of ['/api/security/saml/callback', '/api/security/v1/saml']) { + // For a serverless build, do not register deprecated versioned routes + for (const path of [ + '/api/security/saml/callback', + ...(buildFlavor !== 'serverless' ? ['/api/security/v1/saml'] : []), + ]) { router.post( { path, diff --git a/x-pack/plugins/security/server/routes/authorization/index.ts b/x-pack/plugins/security/server/routes/authorization/index.ts index b3b29e950d72..0e4cc467f3b1 100644 --- a/x-pack/plugins/security/server/routes/authorization/index.ts +++ b/x-pack/plugins/security/server/routes/authorization/index.ts @@ -12,8 +12,14 @@ import { defineShareSavedObjectPermissionRoutes } from './spaces'; import type { RouteDefinitionParams } from '..'; export function defineAuthorizationRoutes(params: RouteDefinitionParams) { - defineRolesRoutes(params); - definePrivilegesRoutes(params); + // The reset session endpoint is registered with httpResources and should remain public in serverless resetSessionPageRoutes(params); - defineShareSavedObjectPermissionRoutes(params); + defineRolesRoutes(params); // Temporarily allow role APIs (ToDo: move to non-serverless block below) + + // In the serverless environment, roles, privileges, and permissions are managed internally and only + // exposed to users and administrators via control plane UI, eliminating the need for any public HTTP APIs. + if (params.buildFlavor !== 'serverless') { + definePrivilegesRoutes(params); + defineShareSavedObjectPermissionRoutes(params); + } } diff --git a/x-pack/plugins/security/server/routes/index.ts b/x-pack/plugins/security/server/routes/index.ts index ba33ca319cd2..99739208e6b7 100644 --- a/x-pack/plugins/security/server/routes/index.ts +++ b/x-pack/plugins/security/server/routes/index.ts @@ -7,6 +7,7 @@ import type { Observable } from 'rxjs'; +import type { BuildFlavor } from '@kbn/config/src/types'; import type { HttpResources, IBasePath, Logger } from '@kbn/core/server'; import type { KibanaFeature } from '@kbn/features-plugin/server'; import type { PublicMethodsOf } from '@kbn/utility-types'; @@ -54,20 +55,26 @@ export interface RouteDefinitionParams { getUserProfileService: () => UserProfileServiceStartInternal; getAnonymousAccessService: () => AnonymousAccessServiceStart; analyticsService: AnalyticsServiceSetup; + buildFlavor: BuildFlavor; } export function defineRoutes(params: RouteDefinitionParams) { + defineAnalyticsRoutes(params); + defineApiKeysRoutes(params); defineAuthenticationRoutes(params); defineAuthorizationRoutes(params); defineSessionManagementRoutes(params); - defineApiKeysRoutes(params); - defineIndicesRoutes(params); - defineUsersRoutes(params); defineUserProfileRoutes(params); - defineRoleMappingRoutes(params); + defineUsersRoutes(params); // Temporarily allow user APIs (ToDo: move to non-serverless block below) defineViewRoutes(params); - defineDeprecationsRoutes(params); - defineAnonymousAccessRoutes(params); - defineSecurityCheckupGetStateRoutes(params); - defineAnalyticsRoutes(params); + + // In the serverless environment... + if (params.buildFlavor !== 'serverless') { + defineAnonymousAccessRoutes(params); // anonymous access is disabled + defineDeprecationsRoutes(params); // deprecated kibana user roles are not applicable, these HTTP APIs are not needed + defineIndicesRoutes(params); // the ES privileges form used to help define roles (only consumer) is disabled, so there is no need for these HTTP APIs + defineRoleMappingRoutes(params); // role mappings are managed internally, based on configurations in control plane, these HTTP APIs are not needed + defineSecurityCheckupGetStateRoutes(params); // security checkup is not applicable, these HTTP APIs are not needed + // defineUsersRoutes(params); // the native realm is not enabled (there is only Elastic cloud SAML), no user HTTP API routes are needed + } } diff --git a/x-pack/plugins/security/server/routes/session_management/index.ts b/x-pack/plugins/security/server/routes/session_management/index.ts index 041feea8a62f..c095a7740997 100644 --- a/x-pack/plugins/security/server/routes/session_management/index.ts +++ b/x-pack/plugins/security/server/routes/session_management/index.ts @@ -13,5 +13,12 @@ import type { RouteDefinitionParams } from '..'; export function defineSessionManagementRoutes(params: RouteDefinitionParams) { defineSessionInfoRoutes(params); defineSessionExtendRoutes(params); - defineInvalidateSessionsRoutes(params); + + // The invalidate session API was introduced to address situations where the session index + // could grow rapidly - when session timeouts are disabled, or with anonymous access. + // In the serverless environment, sessions timeouts are always be enabled, and there is no + // anonymous access. This eliminates the need for an invalidate session HTTP API. + if (params.buildFlavor !== 'serverless') { + defineInvalidateSessionsRoutes(params); + } } diff --git a/x-pack/plugins/security/server/routes/views/index.test.ts b/x-pack/plugins/security/server/routes/views/index.test.ts index 755a5a1202c7..f5f0296a39c1 100644 --- a/x-pack/plugins/security/server/routes/views/index.test.ts +++ b/x-pack/plugins/security/server/routes/views/index.test.ts @@ -12,6 +12,7 @@ describe('View routes', () => { it('does not register Login routes if both `basic` and `token` providers are disabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { providers: { pki: { pki1: { order: 0 } } } }, + accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -19,12 +20,12 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ - "/security/access_agreement", "/security/account", + "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/internal/security/capture-url", + "/security/access_agreement", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` @@ -37,6 +38,7 @@ describe('View routes', () => { it('registers Login routes if `basic` provider is enabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { providers: { basic: { basic1: { order: 0 } } } }, + accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -44,19 +46,19 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ - "/login", - "/security/access_agreement", "/security/account", + "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/internal/security/capture-url", + "/security/access_agreement", + "/login", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` Array [ - "/internal/security/login_state", "/internal/security/access_agreement/state", + "/internal/security/login_state", ] `); }); @@ -64,6 +66,7 @@ describe('View routes', () => { it('registers Login routes if `token` provider is enabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { providers: { token: { token1: { order: 0 } } } }, + accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -71,19 +74,19 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ - "/login", - "/security/access_agreement", "/security/account", + "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", - "/internal/security/capture-url", + "/security/access_agreement", + "/login", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` Array [ - "/internal/security/login_state", "/internal/security/access_agreement/state", + "/internal/security/login_state", ] `); }); @@ -91,6 +94,7 @@ describe('View routes', () => { it('registers Login routes if Login Selector is enabled even if both `token` and `basic` providers are not enabled', () => { const routeParamsMock = routeDefinitionParamsMock.create({ authc: { selector: { enabled: true }, providers: { pki: { pki1: { order: 0 } } } }, + accessAgreement: { message: 'some-message' }, }); defineViewRoutes(routeParamsMock); @@ -98,19 +102,44 @@ describe('View routes', () => { expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) .toMatchInlineSnapshot(` Array [ - "/login", - "/security/access_agreement", "/security/account", + "/internal/security/capture-url", "/security/logged_out", "/logout", "/security/overwritten_session", + "/security/access_agreement", + "/login", + ] + `); + expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` + Array [ + "/internal/security/access_agreement/state", + "/internal/security/login_state", + ] + `); + }); + + it('does not register access agreement routes if access agreement is not enabled', () => { + const routeParamsMock = routeDefinitionParamsMock.create({ + authc: { providers: { basic: { basic1: { order: 0 } } } }, + }); + + defineViewRoutes(routeParamsMock); + + expect(routeParamsMock.httpResources.register.mock.calls.map(([{ path }]) => path)) + .toMatchInlineSnapshot(` + Array [ + "/security/account", "/internal/security/capture-url", + "/security/logged_out", + "/logout", + "/security/overwritten_session", + "/login", ] `); expect(routeParamsMock.router.get.mock.calls.map(([{ path }]) => path)).toMatchInlineSnapshot(` Array [ "/internal/security/login_state", - "/internal/security/access_agreement/state", ] `); }); diff --git a/x-pack/plugins/security/server/routes/views/index.ts b/x-pack/plugins/security/server/routes/views/index.ts index f1efa4611dc5..c9fbb3b1bc36 100644 --- a/x-pack/plugins/security/server/routes/views/index.ts +++ b/x-pack/plugins/security/server/routes/views/index.ts @@ -15,17 +15,23 @@ import { defineOverwrittenSessionRoutes } from './overwritten_session'; import type { RouteDefinitionParams } from '..'; export function defineViewRoutes(params: RouteDefinitionParams) { + defineAccountManagementRoutes(params); + defineCaptureURLRoutes(params); + defineLoggedOutRoutes(params); + defineLogoutRoutes(params); + defineOverwrittenSessionRoutes(params); + + if ( + params.config.accessAgreement?.message || + params.config.authc.sortedProviders.some(({ hasAccessAgreement }) => hasAccessAgreement) + ) { + defineAccessAgreementRoutes(params); + } + if ( params.config.authc.selector.enabled || params.config.authc.sortedProviders.some(({ type }) => type === 'basic' || type === 'token') ) { defineLoginRoutes(params); } - - defineAccessAgreementRoutes(params); - defineAccountManagementRoutes(params); - defineLoggedOutRoutes(params); - defineLogoutRoutes(params); - defineOverwrittenSessionRoutes(params); - defineCaptureURLRoutes(params); } diff --git a/x-pack/plugins/spaces/server/plugin.ts b/x-pack/plugins/spaces/server/plugin.ts index cb8b42f343ba..893d3db22d70 100644 --- a/x-pack/plugins/spaces/server/plugin.ts +++ b/x-pack/plugins/spaces/server/plugin.ts @@ -103,7 +103,7 @@ export class SpacesPlugin private defaultSpaceService?: DefaultSpaceService; - constructor(initializerContext: PluginInitializerContext) { + constructor(private readonly initializerContext: PluginInitializerContext) { this.config$ = initializerContext.config.create(); this.log = initializerContext.logger.get(); this.spacesService = new SpacesService(); @@ -148,18 +148,21 @@ export class SpacesPlugin logger: this.log, }); - const externalRouter = core.http.createRouter(); - initExternalSpacesApi({ - externalRouter, - log: this.log, - getStartServices: core.getStartServices, - getSpacesService, - usageStatsServicePromise, - }); + const router = core.http.createRouter(); + + initExternalSpacesApi( + { + router, + log: this.log, + getStartServices: core.getStartServices, + getSpacesService, + usageStatsServicePromise, + }, + this.initializerContext.env.packageInfo.buildFlavor + ); - const internalRouter = core.http.createRouter(); initInternalSpacesApi({ - internalRouter, + router, getSpacesService, }); diff --git a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts index 4018cfa11b7b..772231679907 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.test.ts @@ -80,7 +80,7 @@ describe('copy to space', () => { }); initCopyToSpacesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts index 7faf03ea60b5..0c866de5bde6 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/copy_to_space.ts @@ -24,10 +24,10 @@ const areObjectsUnique = (objects: SavedObjectIdentifier[]) => _.uniqBy(objects, (o: SavedObjectIdentifier) => `${o.type}:${o.id}`).length === objects.length; export function initCopyToSpacesApi(deps: ExternalRouteDeps) { - const { externalRouter, getSpacesService, usageStatsServicePromise, getStartServices } = deps; + const { router, getSpacesService, usageStatsServicePromise, getStartServices } = deps; const usageStatsClientPromise = usageStatsServicePromise.then(({ getClient }) => getClient()); - externalRouter.post( + router.post( { path: '/api/spaces/_copy_saved_objects', options: { @@ -137,7 +137,7 @@ export function initCopyToSpacesApi(deps: ExternalRouteDeps) { }) ); - externalRouter.post( + router.post( { path: '/api/spaces/_resolve_copy_saved_objects_errors', options: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts b/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts index 02792389424d..09ed757a5df7 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/delete.test.ts @@ -62,7 +62,7 @@ describe('Spaces Public API', () => { }); initDeleteSpacesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/delete.ts b/x-pack/plugins/spaces/server/routes/api/external/delete.ts index b2f0d71d34a2..63679f49847b 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/delete.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/delete.ts @@ -15,9 +15,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initDeleteSpacesApi(deps: ExternalRouteDeps) { - const { externalRouter, log, getSpacesService } = deps; + const { router, log, getSpacesService } = deps; - externalRouter.delete( + router.delete( { path: '/api/spaces/space/{id}', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts index 6af58c124be0..ed7c403182ab 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.test.ts @@ -65,7 +65,7 @@ describe('_disable_legacy_url_aliases', () => { }); initDisableLegacyUrlAliasesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts index ff523ea99c06..725d0c87612f 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/disable_legacy_url_aliases.ts @@ -12,10 +12,10 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initDisableLegacyUrlAliasesApi(deps: ExternalRouteDeps) { - const { externalRouter, getSpacesService, usageStatsServicePromise } = deps; + const { router, getSpacesService, usageStatsServicePromise } = deps; const usageStatsClientPromise = usageStatsServicePromise.then(({ getClient }) => getClient()); - externalRouter.post( + router.post( { path: '/api/spaces/_disable_legacy_url_aliases', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/get.test.ts b/x-pack/plugins/spaces/server/routes/api/external/get.test.ts index 43ac45ec3c4c..34c1ed01e0ce 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get.test.ts @@ -61,7 +61,7 @@ describe('GET space', () => { }); initGetSpaceApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/get.ts b/x-pack/plugins/spaces/server/routes/api/external/get.ts index a26cfe0211d1..ce89aac5fe18 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get.ts @@ -13,9 +13,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetSpaceApi(deps: ExternalRouteDeps) { - const { externalRouter, getSpacesService } = deps; + const { router, getSpacesService } = deps; - externalRouter.get( + router.get( { path: '/api/spaces/space/{id}', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts b/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts index 8fa87bf5ffa4..93eab7a52b81 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_all.test.ts @@ -62,7 +62,7 @@ describe('GET /spaces/space', () => { }); initGetAllSpacesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_all.ts b/x-pack/plugins/spaces/server/routes/api/external/get_all.ts index 06d629a19456..c1dc24caf151 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_all.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_all.ts @@ -13,9 +13,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetAllSpacesApi(deps: ExternalRouteDeps) { - const { externalRouter, log, getSpacesService } = deps; + const { router, log, getSpacesService } = deps; - externalRouter.get( + router.get( { path: '/api/spaces/space', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts index daa957c04d11..c9ea0d71c11b 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.test.ts @@ -61,7 +61,7 @@ describe('get shareable references', () => { spacesClientService: clientServiceStart, }); initGetShareableReferencesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts index ec5b0ce82ece..f9b18961fae5 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/get_shareable_references.ts @@ -12,9 +12,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetShareableReferencesApi(deps: ExternalRouteDeps) { - const { externalRouter, getStartServices } = deps; + const { router, getStartServices } = deps; - externalRouter.post( + router.post( { path: '/api/spaces/_get_shareable_references', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/index.ts b/x-pack/plugins/spaces/server/routes/api/external/index.ts index 8716f63a5657..290807f9b5f4 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/index.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { BuildFlavor } from '@kbn/config/src/types'; import type { CoreSetup, Logger } from '@kbn/core/server'; import { initCopyToSpacesApi } from './copy_to_space'; @@ -21,21 +22,28 @@ import type { SpacesRouter } from '../../../types'; import type { UsageStatsServiceSetup } from '../../../usage_stats'; export interface ExternalRouteDeps { - externalRouter: SpacesRouter; + router: SpacesRouter; getStartServices: CoreSetup['getStartServices']; getSpacesService: () => SpacesServiceStart; usageStatsServicePromise: Promise; log: Logger; } -export function initExternalSpacesApi(deps: ExternalRouteDeps) { - initDeleteSpacesApi(deps); +export function initExternalSpacesApi(deps: ExternalRouteDeps, buildFlavor: BuildFlavor) { + // These two routes are always registered, internal in serverless by default initGetSpaceApi(deps); initGetAllSpacesApi(deps); - initPostSpacesApi(deps); - initPutSpacesApi(deps); - initCopyToSpacesApi(deps); - initUpdateObjectsSpacesApi(deps); - initGetShareableReferencesApi(deps); - initDisableLegacyUrlAliasesApi(deps); + + // In the serverless environment, Spaces are enabled but are effectively hidden from the user. We + // do not support more than 1 space: the default space. These HTTP APIs for creating, deleting, + // updating, and manipulating saved objects across multiple spaces are not needed. + if (buildFlavor !== 'serverless') { + initPutSpacesApi(deps); + initDeleteSpacesApi(deps); + initPostSpacesApi(deps); + initCopyToSpacesApi(deps); + initUpdateObjectsSpacesApi(deps); + initGetShareableReferencesApi(deps); + initDisableLegacyUrlAliasesApi(deps); + } } diff --git a/x-pack/plugins/spaces/server/routes/api/external/post.test.ts b/x-pack/plugins/spaces/server/routes/api/external/post.test.ts index 01c08eca85ec..88c763f31c0b 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/post.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/post.test.ts @@ -62,7 +62,7 @@ describe('Spaces Public API', () => { }); initPostSpacesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/post.ts b/x-pack/plugins/spaces/server/routes/api/external/post.ts index 3ea6da647b4f..d8091a0140e0 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/post.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/post.ts @@ -15,9 +15,9 @@ import { spaceSchema } from '../../../lib/space_schema'; import { createLicensedRouteHandler } from '../../lib'; export function initPostSpacesApi(deps: ExternalRouteDeps) { - const { externalRouter, log, getSpacesService } = deps; + const { router, log, getSpacesService } = deps; - externalRouter.post( + router.post( { path: '/api/spaces/space', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/put.test.ts b/x-pack/plugins/spaces/server/routes/api/external/put.test.ts index 126d15268edf..5e1e6077a758 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/put.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/put.test.ts @@ -62,7 +62,7 @@ describe('PUT /api/spaces/space', () => { }); initPutSpacesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/put.ts b/x-pack/plugins/spaces/server/routes/api/external/put.ts index fb9f81857658..753ec8e02892 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/put.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/put.ts @@ -15,9 +15,9 @@ import { spaceSchema } from '../../../lib/space_schema'; import { createLicensedRouteHandler } from '../../lib'; export function initPutSpacesApi(deps: ExternalRouteDeps) { - const { externalRouter, getSpacesService } = deps; + const { router, getSpacesService } = deps; - externalRouter.put( + router.put( { path: '/api/spaces/space/{id}', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts index 4c5fd44a6bb3..195db8148a37 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.test.ts @@ -62,7 +62,7 @@ describe('update_objects_spaces', () => { spacesClientService: clientServiceStart, }); initUpdateObjectsSpacesApi({ - externalRouter: router, + router, getStartServices: async () => [coreStart, {}, {}], log, getSpacesService: () => spacesServiceStart, diff --git a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts index ea95557514d5..5e610c9693ab 100644 --- a/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts +++ b/x-pack/plugins/spaces/server/routes/api/external/update_objects_spaces.ts @@ -14,7 +14,7 @@ import { SPACE_ID_REGEX } from '../../../lib/space_schema'; import { createLicensedRouteHandler } from '../../lib'; export function initUpdateObjectsSpacesApi(deps: ExternalRouteDeps) { - const { externalRouter, getStartServices } = deps; + const { router, getStartServices } = deps; const spacesSchema = schema.arrayOf( schema.string({ @@ -33,7 +33,7 @@ export function initUpdateObjectsSpacesApi(deps: ExternalRouteDeps) { } ); - externalRouter.post( + router.post( { path: '/api/spaces/_update_objects_spaces', validate: { diff --git a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts index 172f1afec53c..afa0f0a99594 100644 --- a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts +++ b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.test.ts @@ -26,7 +26,7 @@ describe('GET /internal/spaces/_active_space', () => { }); initGetActiveSpaceApi({ - internalRouter: router, + router, getSpacesService: () => service.start({ basePath: coreStart.http.basePath, diff --git a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts index feb6d8b59628..2996e7dbc4ed 100644 --- a/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts +++ b/x-pack/plugins/spaces/server/routes/api/internal/get_active_space.ts @@ -10,9 +10,9 @@ import { wrapError } from '../../../lib/errors'; import { createLicensedRouteHandler } from '../../lib'; export function initGetActiveSpaceApi(deps: InternalRouteDeps) { - const { internalRouter, getSpacesService } = deps; + const { router, getSpacesService } = deps; - internalRouter.get( + router.get( { path: '/internal/spaces/_active_space', validate: false, diff --git a/x-pack/plugins/spaces/server/routes/api/internal/index.ts b/x-pack/plugins/spaces/server/routes/api/internal/index.ts index 2f732bfcaf5a..0cf8135d7b71 100644 --- a/x-pack/plugins/spaces/server/routes/api/internal/index.ts +++ b/x-pack/plugins/spaces/server/routes/api/internal/index.ts @@ -10,7 +10,7 @@ import type { SpacesServiceStart } from '../../../spaces_service/spaces_service' import type { SpacesRouter } from '../../../types'; export interface InternalRouteDeps { - internalRouter: SpacesRouter; + router: SpacesRouter; getSpacesService: () => SpacesServiceStart; } diff --git a/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts b/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts index e9305f07f1b0..67c926229601 100644 --- a/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts +++ b/x-pack/plugins/spaces/server/spaces_client/spaces_client.test.ts @@ -18,7 +18,11 @@ const createMockDebugLogger = () => { }; const createMockConfig = ( - mockConfig: ConfigType = { enabled: true, maxSpaces: 1000, allowFeatureVisibility: true } + mockConfig: ConfigType = { + enabled: true, + maxSpaces: 1000, + allowFeatureVisibility: true, + } ) => { return ConfigSchema.validate(mockConfig, { serverless: !mockConfig.allowFeatureVisibility }); }; @@ -209,7 +213,11 @@ describe('#create', () => { total: maxSpaces - 1, } as any); - const mockConfig = createMockConfig({ enabled: true, maxSpaces, allowFeatureVisibility: true }); + const mockConfig = createMockConfig({ + enabled: true, + maxSpaces, + allowFeatureVisibility: true, + }); const client = new SpacesClient(mockDebugLogger, mockConfig, mockCallWithRequestRepository, []); @@ -235,7 +243,11 @@ describe('#create', () => { total: maxSpaces, } as any); - const mockConfig = createMockConfig({ enabled: true, maxSpaces, allowFeatureVisibility: true }); + const mockConfig = createMockConfig({ + enabled: true, + maxSpaces, + allowFeatureVisibility: true, + }); const client = new SpacesClient(mockDebugLogger, mockConfig, mockCallWithRequestRepository, []); diff --git a/x-pack/plugins/spaces/tsconfig.json b/x-pack/plugins/spaces/tsconfig.json index 43ea0f1c6c56..d59e30e3a019 100644 --- a/x-pack/plugins/spaces/tsconfig.json +++ b/x-pack/plugins/spaces/tsconfig.json @@ -31,6 +31,7 @@ "@kbn/core-custom-branding-browser-mocks", "@kbn/core-custom-branding-common", "@kbn/shared-ux-link-redirect-app", + "@kbn/config", ], "exclude": [ "target/**/*", diff --git a/x-pack/test_serverless/api_integration/services/index.ts b/x-pack/test_serverless/api_integration/services/index.ts index 8102eeb9f4c1..06e7c33fd709 100644 --- a/x-pack/test_serverless/api_integration/services/index.ts +++ b/x-pack/test_serverless/api_integration/services/index.ts @@ -12,6 +12,7 @@ import { services as svlSharedServices } from '../../shared/services'; import { SvlCommonApiServiceProvider } from './svl_common_api'; import { AlertingApiProvider } from './alerting_api'; +import { SamlToolsProvider } from './saml_tools'; import { DataViewApiProvider } from './data_view_api'; export const services = { @@ -20,6 +21,7 @@ export const services = { svlCommonApi: SvlCommonApiServiceProvider, alertingApi: AlertingApiProvider, + samlTools: SamlToolsProvider, dataViewApi: DataViewApiProvider, }; diff --git a/x-pack/test_serverless/api_integration/services/saml_tools.ts b/x-pack/test_serverless/api_integration/services/saml_tools.ts new file mode 100644 index 000000000000..4756109fc667 --- /dev/null +++ b/x-pack/test_serverless/api_integration/services/saml_tools.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +// eslint-disable-next-line @kbn/imports/no_boundary_crossing +import { getSAMLResponse } from '@kbn/security-api-integration-helpers/saml/saml_tools'; +import { kbnTestConfig } from '@kbn/test'; + +import { parse as parseCookie } from 'tough-cookie'; + +import { FtrProviderContext } from '../ftr_provider_context'; + +export function SamlToolsProvider({ getService }: FtrProviderContext) { + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const randomness = getService('randomness'); + const svlCommonApi = getService('svlCommonApi'); + + function createSAMLResponse(options = {}) { + return getSAMLResponse({ + destination: `http://localhost:${kbnTestConfig.getPort()}/api/security/saml/callback`, + sessionIndex: String(randomness.naturalNumber()), + ...options, + }); + } + + return { + async login(username: string) { + const samlAuthenticationResponse = await supertestWithoutAuth + .post('/api/security/saml/callback') + .set(svlCommonApi.getCommonRequestHeader()) + .send({ SAMLResponse: await createSAMLResponse({ username }) }); + expect(samlAuthenticationResponse.status).to.equal(302); + expect(samlAuthenticationResponse.header.location).to.equal('/'); + const sessionCookie = parseCookie(samlAuthenticationResponse.header['set-cookie'][0])!; + return { Cookie: sessionCookie.cookieString() }; + }, + }; +} diff --git a/x-pack/test_serverless/api_integration/services/svl_common_api.ts b/x-pack/test_serverless/api_integration/services/svl_common_api.ts index 15b4f15f851f..b23c8f70a309 100644 --- a/x-pack/test_serverless/api_integration/services/svl_common_api.ts +++ b/x-pack/test_serverless/api_integration/services/svl_common_api.ts @@ -36,5 +36,14 @@ export function SvlCommonApiServiceProvider({}: FtrProviderContext) { )}'` ); }, + + assertApiNotFound(body: unknown, status: number) { + expect(body).to.eql({ + statusCode: 404, + error: 'Not Found', + message: 'Not Found', + }); + expect(status).to.eql(404); + }, }; } diff --git a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts new file mode 100644 index 000000000000..be5dc924c839 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('encrypted saved objects', function () { + describe('route access', () => { + describe('disabled', () => { + it('rotate key', async () => { + const { body, status } = await supertest + .post('/api/encrypted_saved_objects/_rotate_key') + .set(svlCommonApi.getCommonRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index.ts index 8296dd0d33c2..3f6751b8e4d0 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index.ts @@ -9,9 +9,19 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('serverless common API', function () { - loadTestFile(require.resolve('./security_users')); + loadTestFile(require.resolve('./encrypted_saved_objects')); + loadTestFile(require.resolve('./security/anonymous')); + loadTestFile(require.resolve('./security/api_keys')); + loadTestFile(require.resolve('./security/authentication')); + loadTestFile(require.resolve('./security/authorization')); + loadTestFile(require.resolve('./security/misc')); + loadTestFile(require.resolve('./security/response_headers')); + loadTestFile(require.resolve('./security/role_mappings')); + loadTestFile(require.resolve('./security/sessions')); + loadTestFile(require.resolve('./security/users')); + loadTestFile(require.resolve('./security/user_profiles')); + loadTestFile(require.resolve('./security/views')); loadTestFile(require.resolve('./spaces')); - loadTestFile(require.resolve('./security_response_headers')); loadTestFile(require.resolve('./rollups')); loadTestFile(require.resolve('./scripted_fields')); loadTestFile(require.resolve('./index_management')); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts new file mode 100644 index 000000000000..0b08aa18ce12 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/anonymous', function () { + describe('route access', () => { + describe('disabled', () => { + it('get access capabilities', async () => { + const { body, status } = await supertest + .get('/internal/security/anonymous_access/capabilities') + .set(svlCommonApi.getCommonRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get access state', async () => { + const { body, status } = await supertest + .get('/internal/security/anonymous_access/state') + .set(svlCommonApi.getCommonRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts new file mode 100644 index 000000000000..256a0fcfe32a --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + let roleMapping: { id: string; name: string; api_key: string; encoded: string }; + + describe('security/api_keys', function () { + describe('route access', () => { + describe('internal', () => { + before(async () => { + const { body, status } = await supertest + .post('/internal/security/api_key') + .set(svlCommonApi.getInternalRequestHeader()) + .send({ + name: 'test', + metadata: {}, + role_descriptors: {}, + }); + expect(status).toBe(200); + roleMapping = body; + }); + + after(async () => { + const { body, status } = await supertest + .get('/internal/security/api_key?isAdmin=true') + .set(svlCommonApi.getInternalRequestHeader()); + + if (status === 200) { + await supertest + .post('/internal/security/api_key/invalidate') + .set(svlCommonApi.getInternalRequestHeader()) + .send({ + apiKeys: body?.apiKeys, + isAdmin: true, + }); + } + }); + + it('create', async () => { + let body: unknown; + let status: number; + const requestBody = { + name: 'create_test', + metadata: {}, + role_descriptors: {}, + }; + + ({ body, status } = await supertest + .post('/internal/security/api_key') + .set(svlCommonApi.getCommonRequestHeader()) + .send(requestBody)); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [post] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .post('/internal/security/api_key') + .set(svlCommonApi.getInternalRequestHeader()) + .send(requestBody)); + // expect success because we're using the internal header + expect(body).toEqual(expect.objectContaining({ name: 'create_test' })); + expect(status).toBe(200); + }); + + it('update', async () => { + let body: unknown; + let status: number; + const requestBody = { + id: roleMapping.id, + metadata: { test: 'value' }, + role_descriptors: {}, + }; + + ({ body, status } = await supertest + .put('/internal/security/api_key') + .set(svlCommonApi.getCommonRequestHeader()) + .send(requestBody)); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [put] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .put('/internal/security/api_key') + .set(svlCommonApi.getInternalRequestHeader()) + .send(requestBody)); + // expect success because we're using the internal header + expect(body).toEqual(expect.objectContaining({ updated: true })); + expect(status).toBe(200); + }); + + it('get all', async () => { + let body: unknown; + let status: number; + + ({ body, status } = await supertest + .get('/internal/security/api_key?isAdmin=true') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/internal/security/api_key?isAdmin=true') + .set(svlCommonApi.getInternalRequestHeader())); + // expect success because we're using the internal header + expect(body).toEqual( + expect.objectContaining({ + apiKeys: expect.arrayContaining([expect.objectContaining({ id: roleMapping.id })]), + }) + ); + expect(status).toBe(200); + }); + + it('get enabled', async () => { + let body: unknown; + let status: number; + + ({ body, status } = await supertest + .get('/internal/security/api_key/_enabled') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/internal/security/api_key/_enabled') + .set(svlCommonApi.getInternalRequestHeader())); + // expect success because we're using the internal header + expect(body).toEqual({ apiKeysEnabled: true }); + expect(status).toBe(200); + }); + + it('invalidate', async () => { + let body: unknown; + let status: number; + const requestBody = { + apiKeys: [ + { + id: roleMapping.id, + name: roleMapping.name, + }, + ], + isAdmin: true, + }; + + ({ body, status } = await supertest + .post('/internal/security/api_key/invalidate') + .set(svlCommonApi.getCommonRequestHeader()) + .send(requestBody)); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [post] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .post('/internal/security/api_key/invalidate') + .set(svlCommonApi.getInternalRequestHeader()) + .send(requestBody)); + // expect success because we're using the internal header + expect(body).toEqual({ + errors: [], + itemsInvalidated: [ + { + id: roleMapping.id, + name: roleMapping.name, + }, + ], + }); + expect(status).toBe(200); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts new file mode 100644 index 000000000000..590adbe267b4 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/authentication', function () { + describe('route access', () => { + describe('disabled', () => { + // ToDo: uncomment when we disable login + // it('login', async () => { + // const { body, status } = await supertest + // .post('/internal/security/login') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + it('logout (deprecated)', async () => { + const { body, status } = await supertest + .get('/api/security/v1/logout') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get current user (deprecated)', async () => { + const { body, status } = await supertest + .get('/internal/security/v1/me') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('acknowledge access agreement', async () => { + const { body, status } = await supertest + .post('/internal/security/access_agreement/acknowledge') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + describe('OIDC', () => { + it('OIDC implicit', async () => { + const { body, status } = await supertest + .get('/api/security/oidc/implicit') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC implicit (deprecated)', async () => { + const { body, status } = await supertest + .get('/api/security/v1/oidc/implicit') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC implicit.js', async () => { + const { body, status } = await supertest + .get('/internal/security/oidc/implicit.js') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC callback', async () => { + const { body, status } = await supertest + .get('/api/security/oidc/callback') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC callback (deprecated)', async () => { + const { body, status } = await supertest + .get('/api/security/v1/oidc') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC login', async () => { + const { body, status } = await supertest + .post('/api/security/oidc/initiate_login') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC login (deprecated)', async () => { + const { body, status } = await supertest + .post('/api/security/v1/oidc') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('OIDC 3rd party login', async () => { + const { body, status } = await supertest + .get('/api/security/oidc/initiate_login') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + + it('SAML callback (deprecated)', async () => { + const { body, status } = await supertest + .post('/api/security/v1/saml') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + + describe('internal', () => { + it('get current user', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .get('/internal/security/me') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/internal/security/me') + .set(svlCommonApi.getInternalRequestHeader())); + // expect success because we're using the internal header + expect(body).toEqual({ + authentication_provider: { name: '__http__', type: 'http' }, + authentication_realm: { name: 'reserved', type: 'reserved' }, + authentication_type: 'realm', + elastic_cloud_user: false, + email: null, + enabled: true, + full_name: null, + lookup_realm: { name: 'reserved', type: 'reserved' }, + metadata: { _reserved: true }, + roles: ['superuser'], + username: 'elastic', + }); + expect(status).toBe(200); + }); + + // ToDo: remove when we disable login + it('login', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .post('/internal/security/login') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [post] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .post('/internal/security/login') + .set(svlCommonApi.getInternalRequestHeader())); + expect(status).not.toBe(404); + }); + }); + + describe('public', () => { + it('logout', async () => { + const { status } = await supertest.get('/api/security/logout'); + expect(status).toBe(302); + }); + + it('SAML callback', async () => { + const { body, status } = await supertest + .post('/api/security/saml/callback') + .set(svlCommonApi.getCommonRequestHeader()) + .send({ + SAMLResponse: '', + }); + + // Should fail with 401 (not 404) because there is no valid SAML response in the request body + expect(body).toEqual({ + error: 'Unauthorized', + message: 'Unauthorized', + statusCode: 401, + }); + expect(status).not.toBe(404); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts new file mode 100644 index 000000000000..d2f5db13da9b --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/authorization.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/authorization', function () { + describe('route access', () => { + describe('disabled', () => { + it('get all privileges', async () => { + const { body, status } = await supertest + .get('/api/security/privileges') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get built-in elasticsearch privileges', async () => { + const { body, status } = await supertest + .get('/internal/security/esPrivileges/builtin') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + // ToDo: Uncomment when we disable role APIs + // it('create/update role', async () => { + // const { body, status } = await supertest + // .put('/api/security/role/test') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('get role', async () => { + // const { body, status } = await supertest + // .get('/api/security/role/superuser') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('get all roles', async () => { + // const { body, status } = await supertest + // .get('/api/security/role') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('delete role', async () => { + // const { body, status } = await supertest + // .delete('/api/security/role/superuser') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + it('get shared saved object permissions', async () => { + const { body, status } = await supertest + .get('/internal/security/_share_saved_object_permissions') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + + // ToDo: remove when we disable role APIs + describe('internal', () => { + it('create/update role', async () => { + const { status } = await supertest + .put('/api/security/role/test') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('get role', async () => { + const { status } = await supertest + .get('/api/security/role/superuser') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('get all roles', async () => { + const { status } = await supertest + .get('/api/security/role') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('delete role', async () => { + const { status } = await supertest + .delete('/api/security/role/superuser') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + }); + + describe('public', () => { + it('reset session page', async () => { + const { status } = await supertest + .get('/internal/security/reset_session_page.js') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts new file mode 100644 index 000000000000..2b49b3a96ab2 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/misc.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/misc', function () { + describe('route access', () => { + describe('disabled', () => { + it('get index fields', async () => { + const { body, status } = await supertest + .get('/internal/security/fields/test') + .set(svlCommonApi.getInternalRequestHeader()) + .send({ params: 'params' }); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('fix deprecated roles', async () => { + const { body, status } = await supertest + .post('/internal/security/deprecations/kibana_user_role/_fix_users') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('fix deprecated role mappings', async () => { + const { body, status } = await supertest + .post('/internal/security/deprecations/kibana_user_role/_fix_role_mappings') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get security checkup state', async () => { + const { body, status } = await supertest + .get('/internal/security/security_checkup/state') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + + describe('internal', () => { + it('get record auth type', async () => { + const { status } = await supertest + .post('/internal/security/analytics/_record_auth_type') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).toBe(200); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/response_headers.ts similarity index 95% rename from x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts rename to x-pack/test_serverless/api_integration/test_suites/common/security/response_headers.ts index 01d1c1b147aa..56a52914cf9a 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/response_headers.ts @@ -6,13 +6,13 @@ */ import expect from 'expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); - describe('security response headers', function () { + describe('security/response_headers', function () { const defaultCSP = `script-src 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'; frame-ancestors 'self'`; const defaultCOOP = 'same-origin'; const defaultPermissionsPolicy = diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts new file mode 100644 index 000000000000..4d1f25cfd772 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/role_mappings.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/role_mappings', function () { + describe('route access', () => { + describe('disabled', () => { + it('create/update role mapping', async () => { + const { body, status } = await supertest + .post('/internal/security/role_mapping/test') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get role mapping', async () => { + const { body, status } = await supertest + .get('/internal/security/role_mapping/test') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get all role mappings', async () => { + const { body, status } = await supertest + .get('/internal/security/role_mapping') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('delete role mapping', async () => { + // this test works because the message for a missing endpoint is different from a missing role mapping + const { body, status } = await supertest + .delete('/internal/security/role_mapping/test') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('role mapping feature check', async () => { + const { body, status } = await supertest + .get('/internal/security/_check_role_mapping_features') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts new file mode 100644 index 000000000000..2148447fa736 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/sessions.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/sessions', function () { + describe('route access', () => { + describe('disabled', () => { + it('invalidate', async () => { + const { body, status } = await supertest + .post('/api/security/session/_invalidate') + .set(svlCommonApi.getInternalRequestHeader()) + .send({ match: 'all' }); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + + describe('internal', () => { + it('get session info', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .get('/internal/security/session') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/internal/security/session') + .set(svlCommonApi.getInternalRequestHeader())); + // expect 204 because there is no session + expect(status).toBe(204); + }); + + it('extend', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .post('/internal/security/session') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [post] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .post('/internal/security/session') + .set(svlCommonApi.getInternalRequestHeader())); + // expect redirect + expect(status).toBe(302); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts new file mode 100644 index 000000000000..79555fd0953f --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/user_profiles.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { kibanaTestUser } from '@kbn/test'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const samlTools = getService('samlTools'); + + describe('security/user_profiles', function () { + describe('route access', () => { + describe('internal', () => { + it('update', async () => { + const { status } = await supertestWithoutAuth + .post(`/internal/security/user_profile/_data`) + .set(svlCommonApi.getInternalRequestHeader()) + .set(await samlTools.login(kibanaTestUser.username)) + .send({ key: 'value' }); + expect(status).not.toBe(404); + }); + + it('get current', async () => { + const { status } = await supertestWithoutAuth + .get(`/internal/security/user_profile`) + .set(svlCommonApi.getInternalRequestHeader()) + .set(await samlTools.login(kibanaTestUser.username)); + expect(status).not.toBe(404); + }); + + it('bulk get', async () => { + const { status } = await supertestWithoutAuth + .get(`/internal/security/user_profile`) + .set(svlCommonApi.getInternalRequestHeader()) + .set(await samlTools.login(kibanaTestUser.username)) + .send({ uids: ['12345678'] }); + expect(status).not.toBe(404); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/users.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/users.ts new file mode 100644 index 000000000000..2feaa8878d1e --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/users.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/users', function () { + describe('route access', () => { + // ToDo: uncomment when we disable user APIs + // describe('disabled', () => { + // it('get', async () => { + // const { body, status } = await supertest + // .get('/internal/security/users/elastic') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('get all', async () => { + // const { body, status } = await supertest + // .get('/internal/security/users') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('create/update', async () => { + // const { body, status } = await supertest + // .post(`/internal/security/users/some_testuser`) + // .set(svlCommonApi.getInternalRequestHeader()) + // .send({ username: 'some_testuser', password: 'testpassword', roles: [] }); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('delete', async () => { + // const { body, status } = await supertest + // .delete(`/internal/security/users/elastic`) + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('disable', async () => { + // const { body, status } = await supertest + // .post(`/internal/security/users/elastic/_disable`) + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('enable', async () => { + // const { body, status } = await supertest + // .post(`/internal/security/users/elastic/_enable`) + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('set password', async () => { + // const { body, status } = await supertest + // .post(`/internal/security/users/{username}/password`) + // .set(svlCommonApi.getInternalRequestHeader()) + // .send({ + // password: 'old_pw', + // newPassword: 'new_pw', + // }); + // svlCommonApi.assertApiNotFound(body, status); + // }); + // }); + + // ToDo: remove when we disable user APIs + describe('internal', () => { + it('get', async () => { + const { status } = await supertest + .get('/internal/security/users/elastic') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('get all', async () => { + const { status } = await supertest + .get('/internal/security/users') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('create/update', async () => { + const { status } = await supertest + .post(`/internal/security/users/some_testuser`) + .set(svlCommonApi.getInternalRequestHeader()) + .send({ username: 'some_testuser', password: 'testpassword', roles: [] }); + expect(status).not.toBe(404); + }); + + it('delete', async () => { + const { status } = await supertest + .delete(`/internal/security/users/elastic`) + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('disable', async () => { + const { status } = await supertest + .post(`/internal/security/users/elastic/_disable`) + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('enable', async () => { + const { status } = await supertest + .post(`/internal/security/users/elastic/_enable`) + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).not.toBe(404); + }); + + it('set password', async () => { + const { status } = await supertest + .post(`/internal/security/users/{username}/password`) + .set(svlCommonApi.getInternalRequestHeader()) + .send({ + password: 'old_pw', + newPassword: 'new_pw', + }); + expect(status).not.toBe(404); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/views.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/views.ts new file mode 100644 index 000000000000..ac1d994252c5 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/views.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertest'); + + describe('security/views', function () { + describe('route access', () => { + describe('disabled', () => { + // ToDo: uncomment these when we disable login routes + // it('login', async () => { + // const { body, status } = await supertest + // .get('/login') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + // it('get login state', async () => { + // const { body, status } = await supertest + // .get('/internal/security/login_state') + // .set(svlCommonApi.getInternalRequestHeader()); + // svlCommonApi.assertApiNotFound(body, status); + // }); + + it('access agreement', async () => { + const { body, status } = await supertest + .get('/security/access_agreement') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + + it('get access agreement state', async () => { + const { body, status } = await supertest + .get('/internal/security/access_agreement/state') + .set(svlCommonApi.getInternalRequestHeader()); + svlCommonApi.assertApiNotFound(body, status); + }); + }); + + describe('public', () => { + it('login', async () => { + const { status } = await supertest + .get('/login') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).toBe(302); + }); + + it('get login state', async () => { + const { status } = await supertest + .get('/internal/security/login_state') + .set(svlCommonApi.getInternalRequestHeader()); + expect(status).toBe(200); + }); + + it('capture URL', async () => { + const { status } = await supertest + .get('/internal/security/capture-url') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + + it('space selector', async () => { + const { status } = await supertest + .get('/spaces/space_selector') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + + it('enter space', async () => { + const { status } = await supertest + .get('/spaces/enter') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(302); + }); + + it('account', async () => { + const { status } = await supertest + .get('/security/account') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + + it('logged out', async () => { + const { status } = await supertest + .get('/security/logged_out') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + + it('logout', async () => { + const { status } = await supertest + .get('/logout') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + + it('overwritten session', async () => { + const { status } = await supertest + .get('/security/overwritten_session') + .set(svlCommonApi.getCommonRequestHeader()); + expect(status).toBe(200); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security_users.ts b/x-pack/test_serverless/api_integration/test_suites/common/security_users.ts deleted file mode 100644 index 2c82e216505b..000000000000 --- a/x-pack/test_serverless/api_integration/test_suites/common/security_users.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const svlCommonApi = getService('svlCommonApi'); - const supertest = getService('supertest'); - - // Test should be unskipped when the API is disabled - // https://github.com/elastic/kibana/issues/161337 - describe.skip('security/users', function () { - it('rejects request to create user', async () => { - const { body, status } = await supertest - .post(`/internal/security/users/some_testuser`) - .set(svlCommonApi.getInternalRequestHeader()) - .send({ username: 'some_testuser', password: 'testpassword', roles: [] }); - - // in a non-serverless environment this would succeed with a 200 - svlCommonApi.assertResponseStatusCode(400, status, body); - }); - }); -} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts b/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts index fcb3dfe84fd1..78c2456f85ca 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/spaces.ts @@ -13,44 +13,200 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); describe('spaces', function () { - it('rejects request to create a space', async () => { - const { body, status } = await supertest - .post('/api/spaces/space') - .set(svlCommonApi.getInternalRequestHeader()) - .send({ - id: 'custom', - name: 'Custom', - disabledFeatures: [], - }); - - // in a non-serverless environment this would succeed with a 200 - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: - 'Unable to create Space, this exceeds the maximum number of spaces set by the xpack.spaces.maxSpaces setting', + describe('route access', () => { + describe('disabled', () => { + it('#delete', async () => { + const { body, status } = await supertest + .delete('/api/spaces/space/default') + .set(svlCommonApi.getCommonRequestHeader()); + + // normally we'd get a 400 bad request if we tried to delete the default space + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#create', async () => { + const { body, status } = await supertest + .post('/api/spaces/space') + .set(svlCommonApi.getCommonRequestHeader()) + .send({ + id: 'custom', + name: 'Custom', + disabledFeatures: [], + }); + + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#update requires internal header', async () => { + const { body, status } = await supertest + .put('/api/spaces/space/default') + .set(svlCommonApi.getCommonRequestHeader()) + .send({ + id: 'default', + name: 'UPDATED!', + disabledFeatures: [], + }); + + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#copyToSpace', async () => { + const { body, status } = await supertest + .post('/api/spaces/_copy_saved_objects') + .set(svlCommonApi.getCommonRequestHeader()); + + // without a request body we would normally a 400 bad request if the endpoint was registered + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#resolveCopyToSpaceErrors', async () => { + const { body, status } = await supertest + .post('/api/spaces/_resolve_copy_saved_objects_errors') + .set(svlCommonApi.getCommonRequestHeader()); + + // without a request body we would normally a 400 bad request if the endpoint was registered + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#updateObjectsSpaces', async () => { + const { body, status } = await supertest + .post('/api/spaces/_update_objects_spaces') + .set(svlCommonApi.getCommonRequestHeader()); + + // without a request body we would normally a 400 bad request if the endpoint was registered + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#getShareableReferences', async () => { + const { body, status } = await supertest + .post('/api/spaces/_get_shareable_references') + .set(svlCommonApi.getCommonRequestHeader()); + + // without a request body we would normally a 400 bad request if the endpoint was registered + svlCommonApi.assertApiNotFound(body, status); + }); + + it('#disableLegacyUrlAliases', async () => { + const { body, status } = await supertest + .post('/api/spaces/_disable_legacy_url_aliases') + .set(svlCommonApi.getCommonRequestHeader()); + + // without a request body we would normally a 400 bad request if the endpoint was registered + svlCommonApi.assertApiNotFound(body, status); + }); }); - expect(status).toBe(400); - }); - it('rejects request to update a space with disabledFeatures', async () => { - const { body, status } = await supertest - .put('/api/spaces/space/default') - .set(svlCommonApi.getInternalRequestHeader()) - .send({ - id: 'custom', - name: 'Custom', - disabledFeatures: ['some-feature'], - }); - - // in a non-serverless environment this would succeed with a 200 - expect(body).toEqual({ - statusCode: 400, - error: 'Bad Request', - message: - 'Unable to update Space, the disabledFeatures array must be empty when xpack.spaces.allowFeatureVisibility setting is disabled', + describe('internal', () => { + it('#get requires internal header', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .get('/api/spaces/space/default') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/api/spaces/space/default') + .set(svlCommonApi.getInternalRequestHeader())); + // expect success because we're using the internal header + expect(body).toEqual( + expect.objectContaining({ + id: 'default', + }) + ); + expect(status).toBe(200); + }); + + it('#getAll requires internal header', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .get('/api/spaces/space') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/api/spaces/space') + .set(svlCommonApi.getInternalRequestHeader())); + // expect success because we're using the internal header + expect(body).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'default', + }), + ]) + ); + expect(status).toBe(200); + }); + + it('#getActiveSpace requires internal header', async () => { + let body: any; + let status: number; + + ({ body, status } = await supertest + .get('/internal/spaces/_active_space') + .set(svlCommonApi.getCommonRequestHeader())); + // expect a rejection because we're not using the internal header + expect(body).toEqual({ + statusCode: 400, + error: 'Bad Request', + message: expect.stringContaining( + 'method [get] exists but is not available with the current configuration' + ), + }); + expect(status).toBe(400); + + ({ body, status } = await supertest + .get('/internal/spaces/_active_space') + .set(svlCommonApi.getInternalRequestHeader())); + // expect success because we're using the internal header + expect(body).toEqual( + expect.objectContaining({ + id: 'default', + }) + ); + expect(status).toBe(200); + }); }); - expect(status).toBe(400); }); + + // TODO: Re-enable test-suite once users can create and update spaces in the Serverless offering. + // it('rejects request to update a space with disabledFeatures', async () => { + // const { body, status } = await supertest + // .put('/api/spaces/space/default') + // .set(svlCommonApi.getInternalRequestHeader()) + // .send({ + // id: 'custom', + // name: 'Custom', + // disabledFeatures: ['some-feature'], + // }); + // + // // in a non-serverless environment this would succeed with a 200 + // expect(body).toEqual({ + // statusCode: 400, + // error: 'Bad Request', + // message: + // 'Unable to update Space, the disabledFeatures array must be empty when xpack.spaces.allowFeatureVisibility setting is disabled', + // }); + // expect(status).toBe(400); + // }); }); } diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts index 7b9d571d8c7c..a8cd016ed615 100644 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts +++ b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts @@ -6,7 +6,6 @@ */ import { request } from '@kbn/security-solution-plugin/public/management/cypress/tasks/common'; -import { isLocalhost } from '@kbn/security-solution-plugin/scripts/endpoint/common/is_localhost'; import type { ServerlessRoleName } from '../../../../../shared/lib'; import { STANDARD_HTTP_HEADERS } from '../../../../../shared/lib/security/default_http_headers'; @@ -32,7 +31,7 @@ const sendApiLoginRequest = ( url: url.toString(), body: { providerType: 'basic', - providerName: isLocalhost(url.hostname) ? 'basic' : 'cloud-basic', + providerName: 'basic', currentURL: '/', params: { username, diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index c2d4fd700263..a9acd07e3aa2 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -23,6 +23,16 @@ export default async () => { elasticsearch: esTestConfig.getUrlParts(), }; + // "Fake" SAML provider + const idpPath = resolve( + __dirname, + '../../test/security_api_integration/plugins/saml_provider/metadata.xml' + ); + const samlIdPPlugin = resolve( + __dirname, + '../../test/security_api_integration/plugins/saml_provider' + ); + return { servers, @@ -31,6 +41,15 @@ export default async () => { serverArgs: [ // HTTP SSL requires setup for Kibana to trust ESS certs, disable for now 'xpack.security.http.ssl.enabled=false', + + 'xpack.security.authc.token.enabled=true', + 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', + `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${idpPath}`, + 'xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=http://www.elastic.co/saml1', + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://localhost:${servers.kibana.port}`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://localhost:${servers.kibana.port}/logout`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://localhost:${servers.kibana.port}/api/security/saml/callback`, + 'xpack.security.authc.realms.saml.cloud-saml-kibana.attributes.principal=urn:oid:0.0.7', ], }, @@ -42,7 +61,7 @@ export default async () => { sourceArgs: ['--no-base-path', '--env.name=development'], serverArgs: [ `--server.restrictInternalApis=true`, - `--server.port=${kbnTestConfig.getPort()}`, + `--server.port=${servers.kibana.port}`, '--status.allowAnonymous=true', // We shouldn't embed credentials into the URL since Kibana requests to Elasticsearch should // either include `kibanaServerTestUser` credentials, or credentials provided by the test @@ -67,6 +86,16 @@ export default async () => { appenders: ['deprecation'], }, ])}`, + // This ensures that we register the Security SAML API endpoints. + // In the real world the SAML config is injected by control plane. + // basic: { 'basic': { order: 0 } }, + `--plugin-path=${samlIdPPlugin}`, + '--xpack.cloud.id=ftr_fake_cloud_id', + '--xpack.security.authc.selector.enabled=false', + `--xpack.security.authc.providers=${JSON.stringify({ + basic: { basic: { order: 0 } }, + saml: { 'cloud-saml-kibana': { order: 1, realm: 'cloud-saml-kibana' } }, + })}`, '--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', `--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`, ], diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index eb69a3788403..b986a6134525 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -47,5 +47,6 @@ "@kbn/core-http-common", "@kbn/data-views-plugin", "@kbn/core-saved-objects-server", + "@kbn/security-api-integration-helpers", ] } From bb489385f1d7f529ba4e5de74e0c2f0dbd83de96 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 25 Aug 2023 11:31:13 -0600 Subject: [PATCH 110/182] Add support for mounting files into containers --- packages/kbn-es/index.ts | 1 + packages/kbn-es/src/cli_commands/docker.ts | 2 + .../kbn-es/src/cli_commands/serverless.ts | 2 + packages/kbn-es/src/paths.ts | 5 ++ packages/kbn-es/src/utils/docker.test.ts | 8 +-- packages/kbn-es/src/utils/docker.ts | 61 +++++++++++++------ packages/kbn-test/index.ts | 2 + packages/kbn-test/src/es/test_es_cluster.ts | 6 ++ .../lib/config/schema.ts | 1 + .../functional_tests/lib/run_elasticsearch.ts | 3 + x-pack/test_serverless/shared/config.base.ts | 9 +-- 11 files changed, 70 insertions(+), 30 deletions(-) diff --git a/packages/kbn-es/index.ts b/packages/kbn-es/index.ts index 025bf3b5f013..694e92e81ccb 100644 --- a/packages/kbn-es/index.ts +++ b/packages/kbn-es/index.ts @@ -12,4 +12,5 @@ export { SYSTEM_INDICES_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER, KIBANA_SERVERLESS_SUPERUSER_PASSWORD, + getDockerFileMountPath, } from './src/utils'; diff --git a/packages/kbn-es/src/cli_commands/docker.ts b/packages/kbn-es/src/cli_commands/docker.ts index 7471b97f98d1..07e4a64263b4 100644 --- a/packages/kbn-es/src/cli_commands/docker.ts +++ b/packages/kbn-es/src/cli_commands/docker.ts @@ -32,6 +32,7 @@ export const docker: Command = { --kill Kill running ES nodes if detected -E Additional key=value settings to pass to Elasticsearch -D Override Docker command + -F Absolute paths for files to mount into container Examples: @@ -53,6 +54,7 @@ export const docker: Command = { alias: { esArgs: 'E', dockerCmd: 'D', + files: 'F', }, string: ['tag', 'image', 'D'], diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index 2e0651977aea..9be9e850a3e6 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -30,6 +30,7 @@ export const serverless: Command = { --kill Kill running ESS nodes if detected --background Start ESS without attaching to the first node's logs -E Additional key=value settings to pass to Elasticsearch + -F Absolute paths for files to mount into containers Examples: @@ -50,6 +51,7 @@ export const serverless: Command = { alias: { basePath: 'base-path', esArgs: 'E', + files: 'F', }, string: ['tag', 'image'], diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 63effda7123c..78e1381dea0b 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -43,3 +43,8 @@ export const ESS_RESOURCES_PATHS = [ ESS_USERS_PATH, ESS_USERS_ROLES_PATH, ]; + +export const ESS_CONFIG_PATH = '/usr/share/elasticsearch/config/'; + +// Files need to be inside config for permissions reasons inside the container +export const ESS_FILES_PATH = `${ESS_CONFIG_PATH}files/`; diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 4c23970e5ef5..e49736ce9ae1 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -341,7 +341,7 @@ describe('resolveEsArgs()', () => { ssl: true, }); - expect(esArgs).toHaveLength(24); + expect(esArgs).toHaveLength(20); expect(esArgs).not.toEqual(expect.arrayContaining(['xpack.security.enabled=false'])); expect(esArgs).toMatchInlineSnapshot(` Array [ @@ -356,16 +356,12 @@ describe('resolveEsArgs()', () => { "--env", "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elasticsearch.p12", "--env", - "xpack.security.http.ssl.keystore.password=storepass", - "--env", "xpack.security.http.ssl.verification_mode=certificate", "--env", "xpack.security.transport.ssl.enabled=true", "--env", "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elasticsearch.p12", "--env", - "xpack.security.transport.ssl.keystore.password=storepass", - "--env", "xpack.security.transport.ssl.verification_mode=certificate", "--env", "xpack.security.operator_privileges.enabled=true", @@ -422,7 +418,7 @@ describe('setupServerlessVolumes()', () => { (path) => !volumeCmd.some((cmd) => cmd.includes(path)) ); - expect(volumeCmd).toHaveLength(16); + expect(volumeCmd).toHaveLength(18); expect(pathsNotIncludedInCmd).toEqual([]); }); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 1eff21fb8f0e..9f48236ff72c 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -9,15 +9,15 @@ import chalk from 'chalk'; import execa from 'execa'; import fs from 'fs'; import Fsp from 'fs/promises'; -import { resolve } from 'path'; +import { resolve, basename, join } from 'path'; import { ToolingLog } from '@kbn/tooling-log'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; +import { kibanaPackageJson as pkg, REPO_ROOT } from '@kbn/repo-info'; import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_RESOURCES_PATHS, ESS_SECRETS_PATH } from '../paths'; +import { ESS_RESOURCES_PATHS, ESS_SECRETS_PATH, ESS_CONFIG_PATH, ESS_FILES_PATH } from '../paths'; interface BaseOptions { tag?: string; @@ -25,6 +25,7 @@ interface BaseOptions { port?: number; ssl?: boolean; kill?: boolean; + files?: string | string[]; } export interface DockerOptions extends EsClusterExecOptions, BaseOptions { @@ -47,7 +48,6 @@ interface ServerlessEsNodeArgs { export const DEFAULT_PORT = 9200; const DOCKER_REGISTRY = 'docker.elastic.co'; -const ESS_CONFIG_PATH = '/usr/share/elasticsearch/config/'; const DOCKER_BASE_CMD = [ 'run', @@ -388,23 +388,32 @@ function getESp12Volume() { return ['--volume', `${ES_P12_PATH}:${ESS_CONFIG_PATH}certs/elasticsearch.p12`]; } +/** + * Removes REPO_ROOT from hostPath. Keep the rest to avoid filename collisions. + * Returns the path where a file will be mounted inside the ES or ESS container. + * /root/kibana/package/foo/bar.json => /usr/share/elasticsearch/files/package/foo/bar.json + */ +export function getDockerFileMountPath(hostPath: string) { + return join(ESS_FILES_PATH, hostPath.replace(REPO_ROOT, '')); +} + /** * Setup local volumes for Serverless ES */ export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) { - const { basePath, clean, ssl } = options; - const volumePath = resolve(basePath, 'stateless'); + const { basePath, clean, ssl, files } = options; + const objectStorePath = resolve(basePath, 'stateless'); - log.info(chalk.bold(`Checking for local serverless ES object store at ${volumePath}`)); + log.info(chalk.bold(`Checking for local serverless ES object store at ${objectStorePath}`)); log.indent(4); - if (clean && fs.existsSync(volumePath)) { + if (clean && fs.existsSync(objectStorePath)) { log.info('Cleaning existing object store.'); - await Fsp.rm(volumePath, { recursive: true, force: true }); + await Fsp.rm(objectStorePath, { recursive: true, force: true }); } - if (clean || !fs.existsSync(volumePath)) { - await Fsp.mkdir(volumePath, { recursive: true }).then(() => + if (clean || !fs.existsSync(objectStorePath)) { + await Fsp.mkdir(objectStorePath, { recursive: true }).then(() => log.info('Created new object store.') ); } else { @@ -412,30 +421,42 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles } // Permissions are set separately from mkdir due to default umask - await Fsp.chmod(volumePath, 0o777).then(() => { + await Fsp.chmod(objectStorePath, 0o777).then(() => { log.info('Setup object store permissions (chmod 777).'); }); log.indent(-4); - const baseCmd = ['--volume', `${basePath}:/objectstore:z`]; + const volumeCmds = ['--volume', `${basePath}:/objectstore:z`]; + + if (files) { + const _files = typeof files === 'string' ? [files] : files; + const fileCmds = _files.reduce((acc, filePath) => { + acc.push('--volume', `${filePath}:${getDockerFileMountPath(filePath)}:z`); + + return acc; + }, []); + + volumeCmds.push(...fileCmds); + } if (ssl) { const essResources = ESS_RESOURCES_PATHS.reduce((acc, path) => { - const fileName = path.split('/').at(-1); - - acc.push('--volume', `${path}:${ESS_CONFIG_PATH}${fileName}`); + acc.push('--volume', `${path}:${ESS_CONFIG_PATH}${basename(path)}`); return acc; }, []); - return baseCmd.concat(getESp12Volume(), essResources, [ + volumeCmds.push( + ...getESp12Volume(), + ...essResources, + '--volume', - `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, - ]); + `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z` + ); } - return baseCmd; + return volumeCmds; } /** diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index 503a7b3b8034..dc4cd7662d48 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -60,3 +60,5 @@ export * from './src/kbn_archiver_cli'; export * from './src/kbn_client'; export * from './src/find_test_plugin_paths'; + +export { getDockerFileMountPath } from '@kbn/es'; diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 703027296d9b..8857df7c41cc 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -147,6 +147,10 @@ export interface CreateTestEsClusterOptions { * Is this a serverless project */ serverless?: boolean; + /** + * Files to mount inside ES containers + */ + files?: string[]; } export function createTestEsCluster< @@ -168,6 +172,7 @@ export function createTestEsCluster< ssl, transportPort, onEarlyExit, + files, } = options; const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`; @@ -231,6 +236,7 @@ export function createTestEsCluster< teardown: true, ssl: true, background: true, + files, }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 03c0cbc07e64..27bdee55da12 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -210,6 +210,7 @@ export const schema = Joi.object() scheme: /https?/, }), }), + files: Joi.array().items(Joi.string()), }) .default(), diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index f701a5640a8e..09e251d70a25 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -47,6 +47,7 @@ function getEsConfig({ const dataArchive: string | undefined = config.get('esTestCluster.dataArchive'); const serverless: boolean = config.get('serverless'); + const files: string[] | undefined = config.get('esTestCluster.files'); return { ssl, @@ -60,6 +61,7 @@ function getEsConfig({ dataArchive, ccsConfig, serverless, + files, }; } @@ -143,6 +145,7 @@ async function startEsNode({ transportPort: config.transportPort, onEarlyExit, serverless: config.serverless, + files: config.files, }); await cluster.start(); diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index a9acd07e3aa2..7a50d2d596fd 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -14,6 +14,7 @@ import { kbnTestConfig, kibanaServiceAccount, kibanaServerlessSuperuser, + getDockerFileMountPath, } from '@kbn/test'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; @@ -38,13 +39,13 @@ export default async () => { esTestCluster: { from: 'serverless', + files: [idpPath], serverArgs: [ - // HTTP SSL requires setup for Kibana to trust ESS certs, disable for now - 'xpack.security.http.ssl.enabled=false', - 'xpack.security.authc.token.enabled=true', 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', - `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${idpPath}`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${getDockerFileMountPath( + idpPath + )}`, 'xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=http://www.elastic.co/saml1', `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://localhost:${servers.kibana.port}`, `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://localhost:${servers.kibana.port}/logout`, From 85b6ca9e5369934392e190443d7e435dc66dc4de Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 25 Aug 2023 20:26:48 +0100 Subject: [PATCH 111/182] chore(NA): enable ssl for shared serverless config --- x-pack/test_serverless/shared/config.base.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 7a50d2d596fd..6705fae7224b 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -16,11 +16,15 @@ import { kibanaServerlessSuperuser, getDockerFileMountPath, } from '@kbn/test'; +import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; export default async () => { const servers = { - kibana: kbnTestConfig.getUrlParts(kibanaServerlessSuperuser), + kibana: { + ...kbnTestConfig.getUrlParts(kibanaServerlessSuperuser), + protocol: 'https', + }, elasticsearch: esTestConfig.getUrlParts(), }; @@ -36,7 +40,9 @@ export default async () => { return { servers, - + browser: { + acceptInsecureCerts: true, + }, esTestCluster: { from: 'serverless', files: [idpPath], @@ -61,6 +67,10 @@ export default async () => { }, sourceArgs: ['--no-base-path', '--env.name=development'], serverArgs: [ + '--server.ssl.enabled=true', + `--server.ssl.key=${KBN_KEY_PATH}`, + `--server.ssl.certificate=${KBN_CERT_PATH}`, + `--server.ssl.certificateAuthorities=${CA_CERT_PATH}`, `--server.restrictInternalApis=true`, `--server.port=${servers.kibana.port}`, '--status.allowAnonymous=true', From 235f53b509335b9e9d68385e361b33976d93f040 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:34:20 +0000 Subject: [PATCH 112/182] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test_serverless/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index b986a6134525..96583bdfe898 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -48,5 +48,6 @@ "@kbn/data-views-plugin", "@kbn/core-saved-objects-server", "@kbn/security-api-integration-helpers", + "@kbn/dev-utils", ] } From 24d9de5b144c410b87ab0584b8887fe195023730 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 25 Aug 2023 13:34:30 -0600 Subject: [PATCH 113/182] Use https for ess --- x-pack/test_serverless/shared/config.base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 6705fae7224b..015935c77f97 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -25,7 +25,7 @@ export default async () => { ...kbnTestConfig.getUrlParts(kibanaServerlessSuperuser), protocol: 'https', }, - elasticsearch: esTestConfig.getUrlParts(), + elasticsearch: { ...esTestConfig.getUrlParts(), protocol: 'https' }, }; // "Fake" SAML provider From 52e0082e5ccbf790a1c1ad62d01d8c7b72b12840 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 25 Aug 2023 13:46:43 -0600 Subject: [PATCH 114/182] Add ES CA --- x-pack/test_serverless/shared/config.base.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 015935c77f97..d40376d7af36 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -83,6 +83,7 @@ export default async () => { ) )}`, `--elasticsearch.serviceAccountToken=${kibanaServiceAccount.token}`, + `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, '--telemetry.sendUsageTo=staging', `--logging.appenders.deprecation=${JSON.stringify({ type: 'console', From 36436fc449d9338837511ce3363f093fc36420e2 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 25 Aug 2023 14:08:11 -0600 Subject: [PATCH 115/182] Add CA to kibana config --- x-pack/test_serverless/shared/config.base.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index d40376d7af36..fa0252975104 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -7,6 +7,7 @@ import { resolve } from 'path'; import { format as formatUrl } from 'url'; +import Fs from 'fs'; import { REPO_ROOT } from '@kbn/repo-info'; import { @@ -24,6 +25,7 @@ export default async () => { kibana: { ...kbnTestConfig.getUrlParts(kibanaServerlessSuperuser), protocol: 'https', + certificateAuthorities: [Fs.readFileSync(CA_CERT_PATH)], }, elasticsearch: { ...esTestConfig.getUrlParts(), protocol: 'https' }, }; From 584e666857322f6ad987b65ffd93d788cce10192 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 25 Aug 2023 21:09:13 +0100 Subject: [PATCH 116/182] Revert "Revert "Allow Kibana to restrict the usage of JWT for a predefined set of routes only. (#163806)"" This reverts commit 033fbbd875dd7d5f89fa3bab2d9eaa43ea16c06a. --- .github/CODEOWNERS | 2 +- .../src/routes/status.ts | 5 +- .../server/routes/stats/stats.ts | 5 +- .../authentication/authenticator.test.ts | 27 ++++- .../server/authentication/authenticator.ts | 8 +- .../authentication/providers/http.test.ts | 108 ++++++++++++++++++ .../server/authentication/providers/http.ts | 33 ++++++ x-pack/plugins/security/server/config.test.ts | 90 +++++++++++++++ x-pack/plugins/security/server/config.ts | 5 + x-pack/plugins/security/server/routes/tags.ts | 6 + .../background_task_utilization.test.ts | 2 + .../routes/background_task_utilization.ts | 3 + .../server/routes/metrics.test.ts | 1 + .../task_manager/server/routes/metrics.ts | 3 + .../packages/helpers/kibana.jsonc | 2 +- .../test_suites/common/index.ts | 1 + .../common/security/authentication_http.ts | 51 +++++++++ x-pack/test_serverless/shared/config.base.ts | 18 +++ 18 files changed, 361 insertions(+), 9 deletions(-) create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 89571e200ea0..911bfb5161dc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -595,7 +595,7 @@ packages/kbn-search-api-panels @elastic/enterprise-search-frontend examples/search_examples @elastic/kibana-data-discovery packages/kbn-search-response-warnings @elastic/kibana-data-discovery x-pack/plugins/searchprofiler @elastic/platform-deployment-management -x-pack/test/security_api_integration/packages/helpers @elastic/kibana-core +x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security x-pack/plugins/security @elastic/kibana-security x-pack/plugins/security_solution_ess @elastic/security-solution x-pack/test/cases_api_integration/common/plugins/security_solution @elastic/response-ops diff --git a/packages/core/status/core-status-server-internal/src/routes/status.ts b/packages/core/status/core-status-server-internal/src/routes/status.ts index 403686bdf268..e06d667b4c78 100644 --- a/packages/core/status/core-status-server-internal/src/routes/status.ts +++ b/packages/core/status/core-status-server-internal/src/routes/status.ts @@ -82,7 +82,10 @@ export const registerStatusRoute = ({ path: '/api/status', options: { authRequired: 'optional', - tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page + // The `api` tag ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page. + // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to + // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. + tags: ['api', 'security:acceptJWT'], access: 'public', // needs to be public to allow access from "system" users like k8s readiness probes. }, validate: { diff --git a/src/plugins/usage_collection/server/routes/stats/stats.ts b/src/plugins/usage_collection/server/routes/stats/stats.ts index 8c32003f3809..6e4a60621603 100644 --- a/src/plugins/usage_collection/server/routes/stats/stats.ts +++ b/src/plugins/usage_collection/server/routes/stats/stats.ts @@ -55,7 +55,10 @@ export function registerStatsRoute({ path: '/api/stats', options: { authRequired: !config.allowAnonymous, - tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page + // The `api` tag ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page. + // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to + // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. + tags: ['api', 'security:acceptJWT'], access: 'public', // needs to be public to allow access from "system" users like metricbeat. }, validate: { diff --git a/x-pack/plugins/security/server/authentication/authenticator.test.ts b/x-pack/plugins/security/server/authentication/authenticator.test.ts index 2de2fdbf4df2..fbc31e588dc5 100644 --- a/x-pack/plugins/security/server/authentication/authenticator.test.ts +++ b/x-pack/plugins/security/server/authentication/authenticator.test.ts @@ -62,12 +62,14 @@ function getMockOptions({ selector, accessAgreementMessage, customLogoutURL, + configContext = {}, }: { providers?: Record | string[]; http?: Partial; selector?: AuthenticatorOptions['config']['authc']['selector']; accessAgreementMessage?: string; customLogoutURL?: string; + configContext?: Record; } = {}) { const auditService = auditServiceMock.create(); auditLogger = auditLoggerMock.create(); @@ -86,10 +88,10 @@ function getMockOptions({ loggers: loggingSystemMock.create(), getServerBaseURL: jest.fn(), config: createConfig( - ConfigSchema.validate({ - authc: { selector, providers, http }, - ...accessAgreementObj, - }), + ConfigSchema.validate( + { authc: { selector, providers, http }, ...accessAgreementObj }, + configContext + ), loggingSystemMock.create().get(), { isTLSEnabled: false } ), @@ -317,6 +319,23 @@ describe('Authenticator', () => { }); }); + it('includes JWT options if specified', () => { + new Authenticator( + getMockOptions({ + providers: { basic: { basic1: { order: 0 } } }, + http: { jwt: { taggedRoutesOnly: true } }, + configContext: { serverless: true }, + }) + ); + + expect( + jest.requireMock('./providers/http').HTTPAuthenticationProvider + ).toHaveBeenCalledWith(expect.anything(), { + supportedSchemes: new Set(['apikey', 'bearer', 'basic']), + jwt: { taggedRoutesOnly: true }, + }); + }); + it('does not include additional schemes if `autoSchemesEnabled` is disabled', () => { new Authenticator( getMockOptions({ diff --git a/x-pack/plugins/security/server/authentication/authenticator.ts b/x-pack/plugins/security/server/authentication/authenticator.ts index b257052178e2..032512cc5bf6 100644 --- a/x-pack/plugins/security/server/authentication/authenticator.ts +++ b/x-pack/plugins/security/server/authentication/authenticator.ts @@ -649,7 +649,13 @@ export class Authenticator { throw new Error(`Provider name "${options.name}" is reserved.`); } - this.providers.set(options.name, new HTTPAuthenticationProvider(options, { supportedSchemes })); + this.providers.set( + options.name, + new HTTPAuthenticationProvider(options, { + supportedSchemes, + jwt: this.options.config.authc.http.jwt, + }) + ); } /** diff --git a/x-pack/plugins/security/server/authentication/providers/http.test.ts b/x-pack/plugins/security/server/authentication/providers/http.test.ts index c1e7ba662c51..90ff62294ff3 100644 --- a/x-pack/plugins/security/server/authentication/providers/http.test.ts +++ b/x-pack/plugins/security/server/authentication/providers/http.test.ts @@ -15,6 +15,7 @@ import { mockAuthenticationProviderOptions } from './base.mock'; import { HTTPAuthenticationProvider } from './http'; import { mockAuthenticatedUser } from '../../../common/model/authenticated_user.mock'; import { securityMock } from '../../mocks'; +import { ROUTE_TAG_ACCEPT_JWT } from '../../routes/tags'; import { AuthenticationResult } from '../authentication_result'; import { DeauthenticationResult } from '../deauthentication_result'; @@ -144,6 +145,113 @@ describe('HTTPAuthenticationProvider', () => { } }); + it('succeeds for JWT authentication if not restricted to tagged routes.', async () => { + const header = 'Bearer header.body.signature'; + const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } }); + const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } }); + + const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); + mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); + mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); + mockOptions.client.asScoped.mockClear(); + + const provider = new HTTPAuthenticationProvider(mockOptions, { + supportedSchemes: new Set(['bearer']), + }); + + await expect(provider.authenticate(request)).resolves.toEqual( + AuthenticationResult.succeeded({ + ...user, + authentication_provider: { type: 'http', name: 'http' }, + }) + ); + + expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); + + expect(request.headers.authorization).toBe(header); + }); + + it('succeeds for non-JWT authentication if JWT restricted to tagged routes.', async () => { + const header = 'Basic xxx'; + const user = mockAuthenticatedUser(); + const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } }); + + const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); + mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); + mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); + mockOptions.client.asScoped.mockClear(); + + const provider = new HTTPAuthenticationProvider(mockOptions, { + supportedSchemes: new Set(['bearer', 'basic']), + jwt: { taggedRoutesOnly: true }, + }); + + await expect(provider.authenticate(request)).resolves.toEqual( + AuthenticationResult.succeeded({ + ...user, + authentication_provider: { type: 'http', name: 'http' }, + }) + ); + + expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); + + expect(request.headers.authorization).toBe(header); + }); + + it('succeeds for JWT authentication if restricted to tagged routes and route is tagged.', async () => { + const header = 'Bearer header.body.signature'; + const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } }); + const request = httpServerMock.createKibanaRequest({ + headers: { authorization: header }, + routeTags: [ROUTE_TAG_ACCEPT_JWT], + }); + + const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); + mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); + mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); + mockOptions.client.asScoped.mockClear(); + + const provider = new HTTPAuthenticationProvider(mockOptions, { + supportedSchemes: new Set(['bearer']), + jwt: { taggedRoutesOnly: true }, + }); + + await expect(provider.authenticate(request)).resolves.toEqual( + AuthenticationResult.succeeded({ + ...user, + authentication_provider: { type: 'http', name: 'http' }, + }) + ); + + expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); + + expect(request.headers.authorization).toBe(header); + }); + + it('fails for JWT authentication if restricted to tagged routes and route is NOT tagged.', async () => { + const header = 'Bearer header.body.signature'; + const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } }); + const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } }); + + const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); + mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user); + mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient); + mockOptions.client.asScoped.mockClear(); + + const provider = new HTTPAuthenticationProvider(mockOptions, { + supportedSchemes: new Set(['bearer']), + jwt: { taggedRoutesOnly: true }, + }); + + await expect(provider.authenticate(request)).resolves.toEqual( + AuthenticationResult.notHandled() + ); + + expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } }); + + expect(request.headers.authorization).toBe(header); + }); + it('fails if authentication via `authorization` header with supported scheme fails.', async () => { const failureReason = new errors.ResponseError(securityMock.createApiResponse({ body: {} })); for (const { schemes, header } of [ diff --git a/x-pack/plugins/security/server/authentication/providers/http.ts b/x-pack/plugins/security/server/authentication/providers/http.ts index 21c2b25d3be8..dd4bf8c40a43 100644 --- a/x-pack/plugins/security/server/authentication/providers/http.ts +++ b/x-pack/plugins/security/server/authentication/providers/http.ts @@ -9,12 +9,22 @@ import type { KibanaRequest } from '@kbn/core/server'; import type { AuthenticationProviderOptions } from './base'; import { BaseAuthenticationProvider } from './base'; +import { ROUTE_TAG_ACCEPT_JWT } from '../../routes/tags'; import { AuthenticationResult } from '../authentication_result'; import { DeauthenticationResult } from '../deauthentication_result'; import { HTTPAuthorizationHeader } from '../http_authentication'; +/** + * A type-string of the Elasticsearch JWT realm. + */ +const JWT_REALM_TYPE = 'jwt'; + interface HTTPAuthenticationProviderOptions { supportedSchemes: Set; + jwt?: { + // When set, only routes marked with `ROUTE_TAG_ACCEPT_JWT` tag will accept JWT as a means of authentication. + taggedRoutesOnly: boolean; + }; } /** @@ -32,6 +42,11 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider { */ private readonly supportedSchemes: Set; + /** + * Options relevant to the JWT authentication. + */ + private readonly jwt: HTTPAuthenticationProviderOptions['jwt']; + constructor( protected readonly options: Readonly, httpOptions: Readonly @@ -44,6 +59,7 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider { this.supportedSchemes = new Set( [...httpOptions.supportedSchemes].map((scheme) => scheme.toLowerCase()) ); + this.jwt = httpOptions.jwt; } /** @@ -79,6 +95,23 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider { this.logger.debug( `Request to ${request.url.pathname}${request.url.search} has been authenticated via authorization header with "${authorizationHeader.scheme}" scheme.` ); + + // If Kibana is configured to restrict JWT authentication only to selected routes, ensure that the route is marked + // with the `ROUTE_TAG_ACCEPT_JWT` tag to bypass that restriction. + if ( + user.authentication_realm.type === JWT_REALM_TYPE && + this.jwt?.taggedRoutesOnly && + !request.route.options.tags.includes(ROUTE_TAG_ACCEPT_JWT) + ) { + // Log a portion of the JWT signature to make debugging easier. + const jwtExcerpt = authorizationHeader.credentials.slice(-10); + this.logger.error( + `Attempted to authenticate with JWT credentials (…${jwtExcerpt}) against ${request.url.pathname}${request.url.search}, but it's not allowed. ` + + `Ensure that the route is defined with the "${ROUTE_TAG_ACCEPT_JWT}" tag.` + ); + return AuthenticationResult.notHandled(); + } + return AuthenticationResult.succeeded(user); } catch (err) { this.logger.debug( diff --git a/x-pack/plugins/security/server/config.test.ts b/x-pack/plugins/security/server/config.test.ts index 47b16b575279..04b16aebab9c 100644 --- a/x-pack/plugins/security/server/config.test.ts +++ b/x-pack/plugins/security/server/config.test.ts @@ -183,6 +183,68 @@ describe('config schema', () => { "showNavLinks": true, } `); + + expect(ConfigSchema.validate({}, { serverless: true, dist: true })).toMatchInlineSnapshot(` + Object { + "audit": Object { + "enabled": false, + }, + "authc": Object { + "http": Object { + "autoSchemesEnabled": true, + "enabled": true, + "jwt": Object { + "taggedRoutesOnly": true, + }, + "schemes": Array [ + "apikey", + "bearer", + ], + }, + "providers": Object { + "anonymous": undefined, + "basic": Object { + "basic": Object { + "accessAgreement": undefined, + "description": undefined, + "enabled": true, + "hint": undefined, + "icon": undefined, + "order": 0, + "session": Object { + "idleTimeout": undefined, + "lifespan": undefined, + }, + "showInSelector": true, + }, + }, + "kerberos": undefined, + "oidc": undefined, + "pki": undefined, + "saml": undefined, + "token": undefined, + }, + "selector": Object {}, + }, + "cookieName": "sid", + "enabled": true, + "loginAssistanceMessage": "", + "public": Object {}, + "secureCookies": false, + "session": Object { + "cleanupInterval": "PT1H", + "idleTimeout": "P3D", + "lifespan": "P30D", + }, + "showInsecureClusterWarning": true, + "showNavLinks": true, + "ui": Object { + "roleManagementEnabled": true, + "roleMappingManagementEnabled": true, + "userManagementEnabled": true, + }, + } + `); }); it('should throw error if xpack.security.encryptionKey is less than 32 characters', () => { @@ -1412,6 +1474,34 @@ describe('config schema', () => { }); }); + describe('authc.http', () => { + it('should not allow xpack.security.authc.http.jwt.* to be configured outside of the serverless context', () => { + expect(() => + ConfigSchema.validate( + { authc: { http: { jwt: { taggedRoutesOnly: false } } } }, + { serverless: false } + ) + ).toThrowErrorMatchingInlineSnapshot( + `"[authc.http.jwt]: a value wasn't expected to be present"` + ); + }); + + it('should allow xpack.security.authc.http.jwt.* to be configured inside of the serverless context', () => { + expect( + ConfigSchema.validate( + { authc: { http: { jwt: { taggedRoutesOnly: false } } } }, + { serverless: true } + ).ui + ).toMatchInlineSnapshot(` + Object { + "roleManagementEnabled": true, + "roleMappingManagementEnabled": true, + "userManagementEnabled": true, + } + `); + }); + }); + describe('ui', () => { it('should not allow xpack.security.ui.* to be configured outside of the serverless context', () => { expect(() => diff --git a/x-pack/plugins/security/server/config.ts b/x-pack/plugins/security/server/config.ts index 15cb6461b97b..a5483b4e70ba 100644 --- a/x-pack/plugins/security/server/config.ts +++ b/x-pack/plugins/security/server/config.ts @@ -279,6 +279,11 @@ export const ConfigSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), autoSchemesEnabled: schema.boolean({ defaultValue: true }), schemes: schema.arrayOf(schema.string(), { defaultValue: ['apikey', 'bearer'] }), + jwt: offeringBasedSchema({ + serverless: schema.object({ + taggedRoutesOnly: schema.boolean({ defaultValue: true }), + }), + }), }), }), audit: schema.object({ diff --git a/x-pack/plugins/security/server/routes/tags.ts b/x-pack/plugins/security/server/routes/tags.ts index 090c04d29757..a6ffd49d53a5 100644 --- a/x-pack/plugins/security/server/routes/tags.ts +++ b/x-pack/plugins/security/server/routes/tags.ts @@ -25,3 +25,9 @@ export const ROUTE_TAG_CAN_REDIRECT = 'security:canRedirect'; * parties, require special handling. */ export const ROUTE_TAG_AUTH_FLOW = 'security:authFlow'; + +/** + * If `xpack.security.authc.http.jwt.taggedRoutesOnly` flag is set, then only routes marked with this tag will accept + * JWT as a means of authentication. + */ +export const ROUTE_TAG_ACCEPT_JWT = 'security:acceptJWT'; diff --git a/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts b/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts index e70c78b8e212..322060b4f9b6 100644 --- a/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts +++ b/x-pack/plugins/task_manager/server/routes/background_task_utilization.test.ts @@ -57,11 +57,13 @@ describe('backgroundTaskUtilizationRoute', () => { `"/internal/task_manager/_background_task_utilization"` ); expect(config1.options?.authRequired).toEqual(true); + expect(config1.options?.tags).toEqual(['security:acceptJWT']); const [config2] = router.get.mock.calls[1]; expect(config2.path).toMatchInlineSnapshot(`"/api/task_manager/_background_task_utilization"`); expect(config2.options?.authRequired).toEqual(true); + expect(config2.options?.tags).toEqual(['security:acceptJWT']); }); it(`sets "authRequired" to false when config.unsafe.authenticate_background_task_utilization is set to false`, async () => { diff --git a/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts b/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts index 38b1ce9966f3..b72b8ad5a704 100644 --- a/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts +++ b/x-pack/plugins/task_manager/server/routes/background_task_utilization.ts @@ -117,6 +117,9 @@ export function backgroundTaskUtilizationRoute( options: { access: 'public', // access must be public to allow "system" users, like metrics collectors, to access these routes authRequired: routeOption.isAuthenticated ?? true, + // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to + // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. + tags: ['security:acceptJWT'], }, }, async function ( diff --git a/x-pack/plugins/task_manager/server/routes/metrics.test.ts b/x-pack/plugins/task_manager/server/routes/metrics.test.ts index a9703aa7548d..172e29d61f11 100644 --- a/x-pack/plugins/task_manager/server/routes/metrics.test.ts +++ b/x-pack/plugins/task_manager/server/routes/metrics.test.ts @@ -28,6 +28,7 @@ describe('metricsRoute', () => { const [config] = router.get.mock.calls[0]; expect(config.path).toMatchInlineSnapshot(`"/api/task_manager/metrics"`); + expect(config.options?.tags).toEqual(['security:acceptJWT']); }); it('emits resetMetric$ event when route is accessed and reset query param is true', async () => { diff --git a/x-pack/plugins/task_manager/server/routes/metrics.ts b/x-pack/plugins/task_manager/server/routes/metrics.ts index 737f2b44fd79..692227899979 100644 --- a/x-pack/plugins/task_manager/server/routes/metrics.ts +++ b/x-pack/plugins/task_manager/server/routes/metrics.ts @@ -48,6 +48,9 @@ export function metricsRoute(params: MetricsRouteParams) { path: `/api/task_manager/metrics`, options: { access: 'public', + // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to + // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly. + tags: ['security:acceptJWT'], }, // Uncomment when we determine that we can restrict API usage to Global admins based on telemetry // options: { tags: ['access:taskManager'] }, diff --git a/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc b/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc index 6eeec0c14c5c..accbad462016 100644 --- a/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc +++ b/x-pack/test/security_api_integration/packages/helpers/kibana.jsonc @@ -1,6 +1,6 @@ { "type": "shared-common", "id": "@kbn/security-api-integration-helpers", - "owner": "@elastic/kibana-core", + "owner": "@elastic/kibana-security", "devOnly": true } diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index.ts index 3f6751b8e4d0..4ea6b50bb8f2 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index.ts @@ -13,6 +13,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./security/anonymous')); loadTestFile(require.resolve('./security/api_keys')); loadTestFile(require.resolve('./security/authentication')); + loadTestFile(require.resolve('./security/authentication_http')); loadTestFile(require.resolve('./security/authorization')); loadTestFile(require.resolve('./security/misc')); loadTestFile(require.resolve('./security/response_headers')); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts new file mode 100644 index 000000000000..1555ebe352df --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication_http.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const svlCommonApi = getService('svlCommonApi'); + const supertest = getService('supertestWithoutAuth'); + + describe('security/authentication/http', function () { + it('allows JWT HTTP authentication only for selected routes', async () => { + const jsonWebToken = + 'eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwczovL2tpYmFuYS5lbGFzdGljLmNvL2p3dC8iLCJzdWIiOiJlbGFzdGljLWFnZW50IiwiYXVkIjoiZWxhc3RpY3NlYXJjaCIsIm5hbWUiOiJFbGFzdGljIEFnZW50IiwiaWF0Ijo5NDY2ODQ4MDAsImV4cCI6NDA3MDkwODgwMH0.P7RHKZlLskS5DfVRqoVO4ivoIq9rXl2-GW6hhC9NvTSkwphYivcjpTVcyENZvxTTvJJNqcyx6rF3T-7otTTIHBOZIMhZauc5dob-sqcN_mT2htqm3BpSdlJlz60TBq6diOtlNhV212gQCEJMPZj0MNj7kZRj_GsECrTaU7FU0A3HAzkbdx15vQJMKZiFbbQCVI7-X2J0bZzQKIWfMHD-VgHFwOe6nomT-jbYIXtCBDd6fNj1zTKRl-_uzjVqNK-h8YW1h6tE4xvZmXyHQ1-9yNKZIWC7iEaPkBLaBKQulLU5MvW3AtVDUhzm6--5H1J85JH5QhRrnKYRon7ZW5q1AQ'; + + // Check 5 routes that are currently known to accept JWT as a means of authentication. + for (const allowedPath of [ + '/api/status', + '/api/stats', + '/api/task_manager/_background_task_utilization', + '/internal/task_manager/_background_task_utilization', + '/api/task_manager/metrics', + ]) { + await supertest + .get(allowedPath) + .set('Authorization', `Bearer ${jsonWebToken}`) + .set('ES-Client-Authentication', 'SharedSecret my_super_secret') + .set(svlCommonApi.getInternalRequestHeader()) + .expect(200); + } + + // Make sure it's not possible to use JWT to have interactive sessions. + await supertest + .get('/') + .set('Authorization', `Bearer ${jsonWebToken}`) + .set('ES-Client-Authentication', 'SharedSecret my_super_secret') + .expect(401); + + // Make sure it's not possible to use JWT to access any other APIs. + await supertest + .get('/internal/security/me') + .set('Authorization', `Bearer ${jsonWebToken}`) + .set('ES-Client-Authentication', 'SharedSecret my_super_secret') + .set(svlCommonApi.getInternalRequestHeader()) + .expect(401); + }); + }); +} diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 015935c77f97..cdc24462408c 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -38,6 +38,8 @@ export default async () => { '../../test/security_api_integration/plugins/saml_provider' ); + const jwksPath = require.resolve('@kbn/security-api-integration-helpers/oidc/jwks.json'); + return { servers, browser: { @@ -47,6 +49,22 @@ export default async () => { from: 'serverless', files: [idpPath], serverArgs: [ + 'xpack.security.authc.realms.file.file1.order=-100', + + 'xpack.security.authc.realms.jwt.jwt1.order=-98', + `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`, + 'xpack.security.authc.realms.jwt.jwt1.client_authentication.type=shared_secret', + `xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret=my_super_secret`, + `xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/`, + `xpack.security.authc.realms.jwt.jwt1.allowed_subjects=elastic-agent`, + 'xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch', + `xpack.security.authc.realms.jwt.jwt1.allowed_signature_algorithms=[RS256]`, + `xpack.security.authc.realms.jwt.jwt1.claims.principal=sub`, + `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${jwksPath}`, + + // TODO: We should set this flag to `false` as soon as we fully migrate tests to SAML and file realms. + `xpack.security.authc.realms.native.native1.enabled=true`, + `xpack.security.authc.realms.native.native1.order=-97`, 'xpack.security.authc.token.enabled=true', 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${getDockerFileMountPath( From 993ec6f2ce5f7a76ac6ec51d4e42b2e08c8825d0 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 25 Aug 2023 21:17:02 +0100 Subject: [PATCH 117/182] fix(NA): wrong saml setting on shared serverless config --- x-pack/test_serverless/shared/config.base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index ecbe413c0fed..cdacec7acfeb 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -68,7 +68,7 @@ export default async () => { `xpack.security.authc.realms.native.native1.enabled=true`, `xpack.security.authc.realms.native.native1.order=-97`, 'xpack.security.authc.token.enabled=true', - 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', + 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=101', `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${getDockerFileMountPath( idpPath )}`, From 4e7625c0f68844c2c6ef4529fe2b9b88f5c75237 Mon Sep 17 00:00:00 2001 From: Brad White Date: Fri, 25 Aug 2023 14:41:53 -0600 Subject: [PATCH 118/182] Fix jwt PR secrets, file mount --- packages/kbn-es/src/ess_resources/secrets.json | 3 ++- x-pack/test_serverless/shared/config.base.ts | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/kbn-es/src/ess_resources/secrets.json b/packages/kbn-es/src/ess_resources/secrets.json index c19d78f7bd53..ceb7366ee532 100644 --- a/packages/kbn-es/src/ess_resources/secrets.json +++ b/packages/kbn-es/src/ess_resources/secrets.json @@ -5,6 +5,7 @@ }, "string_secrets": { "xpack.security.http.ssl.keystore.secure_password": "storepass", - "xpack.security.transport.ssl.keystore.secure_password": "storepass" + "xpack.security.transport.ssl.keystore.secure_password": "storepass", + "xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret": "my_super_secret" } } \ No newline at end of file diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index cdacec7acfeb..431f89cf7e88 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -49,23 +49,21 @@ export default async () => { }, esTestCluster: { from: 'serverless', - files: [idpPath], + files: [idpPath, jwksPath], serverArgs: [ 'xpack.security.authc.realms.file.file1.order=-100', 'xpack.security.authc.realms.jwt.jwt1.order=-98', `xpack.security.authc.realms.jwt.jwt1.token_type=access_token`, 'xpack.security.authc.realms.jwt.jwt1.client_authentication.type=shared_secret', - `xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret=my_super_secret`, `xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/`, `xpack.security.authc.realms.jwt.jwt1.allowed_subjects=elastic-agent`, 'xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch', `xpack.security.authc.realms.jwt.jwt1.allowed_signature_algorithms=[RS256]`, `xpack.security.authc.realms.jwt.jwt1.claims.principal=sub`, - `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${jwksPath}`, + `xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=${getDockerFileMountPath(jwksPath)}`, - // TODO: We should set this flag to `false` as soon as we fully migrate tests to SAML and file realms. - `xpack.security.authc.realms.native.native1.enabled=true`, + `xpack.security.authc.realms.native.native1.enabled=false`, `xpack.security.authc.realms.native.native1.order=-97`, 'xpack.security.authc.token.enabled=true', 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=101', From 21dfc6d08a9c74802161f4310d7980935428cad0 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 26 Aug 2023 03:38:37 +0100 Subject: [PATCH 119/182] refact(NA): move kibana dev service account out of @kbn/test --- packages/kbn-dev-utils/index.ts | 1 + .../kbn-dev-utils/src/dev_service_account.ts | 19 +++++++++++++++++++ packages/kbn-test/index.ts | 1 - packages/kbn-test/src/kbn/index.ts | 1 - packages/kbn-test/src/kbn/users.ts | 10 ---------- src/cli/serve/serve.js | 4 ++-- x-pack/test_serverless/shared/config.base.ts | 5 ++--- 7 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 packages/kbn-dev-utils/src/dev_service_account.ts diff --git a/packages/kbn-dev-utils/index.ts b/packages/kbn-dev-utils/index.ts index 86bdafebccf9..2faf0000fde2 100644 --- a/packages/kbn-dev-utils/index.ts +++ b/packages/kbn-dev-utils/index.ts @@ -19,6 +19,7 @@ export { KBN_P12_PATH, KBN_P12_PASSWORD, } from './src/certs'; +export * from './src/dev_service_account'; export * from './src/axios'; export * from './src/plugin_list'; export * from './src/streams'; diff --git a/packages/kbn-dev-utils/src/dev_service_account.ts b/packages/kbn-dev-utils/src/dev_service_account.ts new file mode 100644 index 000000000000..efcd51b3f155 --- /dev/null +++ b/packages/kbn-dev-utils/src/dev_service_account.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +const env = process.env; + +/** + * `kibana-dev` service account token for connecting to ESS + * See packages/kbn-es/src/ess_resources/README.md + */ +export const kibanaDevServiceAccount = { + token: + env.TEST_KIBANA_SERVICE_ACCOUNT_TOKEN || + 'AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA', +}; diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index dc4cd7662d48..7f3a87ee45c6 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -38,7 +38,6 @@ export { kibanaTestUser, adminTestUser, systemIndicesSuperuser, - kibanaServiceAccount, kibanaServerlessSuperuser, } from './src/kbn'; diff --git a/packages/kbn-test/src/kbn/index.ts b/packages/kbn-test/src/kbn/index.ts index 414f7d63554c..0c0929e918f3 100644 --- a/packages/kbn-test/src/kbn/index.ts +++ b/packages/kbn-test/src/kbn/index.ts @@ -11,6 +11,5 @@ export { kibanaServerTestUser, adminTestUser, systemIndicesSuperuser, - kibanaServiceAccount, kibanaServerlessSuperuser, } from './users'; diff --git a/packages/kbn-test/src/kbn/users.ts b/packages/kbn-test/src/kbn/users.ts index e1d7cef0f40d..de8a7937d86d 100644 --- a/packages/kbn-test/src/kbn/users.ts +++ b/packages/kbn-test/src/kbn/users.ts @@ -37,16 +37,6 @@ export const systemIndicesSuperuser = { password: env.TEST_ES_PASS || 'changeme', }; -/** - * `kibana-dev` service account token for connecting to ESS - * See packages/kbn-es/src/ess_resources/README.md - */ -export const kibanaServiceAccount = { - token: - env.TEST_KIBANA_SERVICE_ACCOUNT_TOKEN || - 'AAEAAWVsYXN0aWMva2liYW5hL2tpYmFuYS1kZXY6VVVVVVVVTEstKiBaNA', -}; - export const kibanaServerlessSuperuser = { username: KIBANA_SERVERLESS_SUPERUSER, password: KIBANA_SERVERLESS_SUPERUSER_PASSWORD, diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 3998e07417d1..192ea6e14955 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -15,7 +15,7 @@ import { isKibanaDistributable } from '@kbn/repo-info'; import { readKeystore } from '../keystore/read_keystore'; import { compileConfigStack } from './compile_config_stack'; import { getConfigFromFiles } from '@kbn/config'; -import { kibanaServiceAccount } from '@kbn/test'; +import { kibanaDevServiceAccount } from '@kbn/dev-utils'; const DEV_MODE_PATH = '@kbn/cli-dev-mode'; const DEV_MODE_SUPPORTED = canRequire(DEV_MODE_PATH); @@ -70,7 +70,7 @@ export function applyConfigOverrides(rawConfig, opts, extraCliOptions) { if (opts.dev) { if (opts.serverless) { - set('elasticsearch.serviceAccountToken', kibanaServiceAccount.token); + set('elasticsearch.serviceAccountToken', kibanaDevServiceAccount.token); } if (!has('elasticsearch.serviceAccountToken') && opts.devCredentials !== false) { diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 431f89cf7e88..32d6cf20eb79 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -13,11 +13,10 @@ import { REPO_ROOT } from '@kbn/repo-info'; import { esTestConfig, kbnTestConfig, - kibanaServiceAccount, kibanaServerlessSuperuser, getDockerFileMountPath, } from '@kbn/test'; -import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils'; +import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; export default async () => { @@ -100,7 +99,7 @@ export default async () => { Object.entries(servers.elasticsearch).filter(([key]) => key.toLowerCase() !== 'auth') ) )}`, - `--elasticsearch.serviceAccountToken=${kibanaServiceAccount.token}`, + `--elasticsearch.serviceAccountToken=${kibanaDevServiceAccount.token}`, `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, '--telemetry.sendUsageTo=staging', `--logging.appenders.deprecation=${JSON.stringify({ From cc4098318c17afc1ad3c8250db6b2b7ccf998bf4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 26 Aug 2023 02:54:14 +0000 Subject: [PATCH 120/182] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/cli/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cli/tsconfig.json b/src/cli/tsconfig.json index 3f14fafce07b..ebbbc19f75c7 100644 --- a/src/cli/tsconfig.json +++ b/src/cli/tsconfig.json @@ -17,7 +17,6 @@ "@kbn/config", "@kbn/dev-utils", "@kbn/apm-config-loader", - "@kbn/test", ], "exclude": [ "target/**/*", From d8daf9426059b31e7abf50758906aa98b70e0cd1 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 26 Aug 2023 04:57:06 +0100 Subject: [PATCH 121/182] fix(NA): isolated init of yarn es serverless --ssl --- packages/kbn-es/src/ess_resources/jwks.json | 10 ++++++++++ packages/kbn-es/src/paths.ts | 2 ++ packages/kbn-es/src/utils/docker.ts | 19 +++++++++++++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 packages/kbn-es/src/ess_resources/jwks.json diff --git a/packages/kbn-es/src/ess_resources/jwks.json b/packages/kbn-es/src/ess_resources/jwks.json new file mode 100644 index 000000000000..944705b31416 --- /dev/null +++ b/packages/kbn-es/src/ess_resources/jwks.json @@ -0,0 +1,10 @@ +{ + "keys": [ + { + "kty": "RSA", + "e": "AQAB", + "use": "sig", + "n": "v9-88aGdE4E85PuEycxTA6LkM3TBvNScoeP6A-dd0Myo6-LfBlp1r7BPBWmvi_SC6Zam3U1LE3AekDMwqJg304my0pvh8wOwlmRpgKXDXjvj4s59vdeVNhCB9doIthUABd310o9lyb55fWc_qQYE2LK9AyEjicJswafguH6txV4IwSl13ieZAxni0Ca4CwdzXO1Oi34XjHF8F5x_0puTaQzHn5bPG4fiIJN-pwie0Ba4VEDPO5ca4lLXWVi1bn8xMDTAULrBAXJwDaDdS05KMbc4sPlyQPhtY1gcYvUbozUPYxSWwA7fZgFzV_h-uy_oXf1EXttOxSgog1z3cJzf6Q" + } + ] +} diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 78e1381dea0b..4bd3ea9b7c0c 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -35,6 +35,8 @@ export const ESS_ROLE_MAPPING_PATH = resolve(__dirname, './ess_resources/role_ma export const ESS_SECRETS_PATH = resolve(__dirname, './ess_resources/secrets.json'); +export const ESS_JWKS_PATH = resolve(__dirname, './ess_resources/jwks.json'); + export const ESS_RESOURCES_PATHS = [ ESS_OPERATOR_USERS_PATH, ESS_ROLE_MAPPING_PATH, diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 9f48236ff72c..564f72502256 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -17,7 +17,7 @@ import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_RESOURCES_PATHS, ESS_SECRETS_PATH, ESS_CONFIG_PATH, ESS_FILES_PATH } from '../paths'; +import { ESS_RESOURCES_PATHS, ESS_SECRETS_PATH, ESS_JWKS_PATH, ESS_CONFIG_PATH, ESS_FILES_PATH } from '../paths'; interface BaseOptions { tag?: string; @@ -147,6 +147,18 @@ const DEFAULT_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.transport.ssl.verification_mode', 'certificate'], ['xpack.security.operator_privileges.enabled', 'true'], + + ['xpack.security.authc.realms.jwt.jwt1.client_authentication.type', 'shared_secret'], + + ['xpack.security.authc.realms.jwt.jwt1.order', '-98'], + + ['xpack.security.authc.realms.jwt.jwt1.allowed_issuer', 'https://kibana.elastic.co/jwt/'], + + ['xpack.security.authc.realms.jwt.jwt1.allowed_audiences', 'elasticsearch'], + + ['xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path', `${ESS_CONFIG_PATH}secrets/jwks.json`], + + ['xpack.security.authc.realms.jwt.jwt1.claims.principal', 'sub'] ]; const DOCKER_SSL_ESARGS: Array<[string, string]> = [ @@ -452,7 +464,10 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles ...essResources, '--volume', - `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z` + `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, + + '--volume', + `${ESS_JWKS_PATH}:${ESS_CONFIG_PATH}secrets/jwks.json:z`, ); } From c7beba4fd1de113d3c656014742cdd07b1547d53 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 26 Aug 2023 04:02:11 +0000 Subject: [PATCH 122/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- packages/kbn-es/src/paths.ts | 2 +- packages/kbn-es/src/utils/docker.ts | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/kbn-es/src/paths.ts b/packages/kbn-es/src/paths.ts index 4bd3ea9b7c0c..76cf4271c7ce 100644 --- a/packages/kbn-es/src/paths.ts +++ b/packages/kbn-es/src/paths.ts @@ -35,7 +35,7 @@ export const ESS_ROLE_MAPPING_PATH = resolve(__dirname, './ess_resources/role_ma export const ESS_SECRETS_PATH = resolve(__dirname, './ess_resources/secrets.json'); -export const ESS_JWKS_PATH = resolve(__dirname, './ess_resources/jwks.json'); +export const ESS_JWKS_PATH = resolve(__dirname, './ess_resources/jwks.json'); export const ESS_RESOURCES_PATHS = [ ESS_OPERATOR_USERS_PATH, diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 564f72502256..c5fb46ca7dc5 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -17,7 +17,13 @@ import { ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; -import { ESS_RESOURCES_PATHS, ESS_SECRETS_PATH, ESS_JWKS_PATH, ESS_CONFIG_PATH, ESS_FILES_PATH } from '../paths'; +import { + ESS_RESOURCES_PATHS, + ESS_SECRETS_PATH, + ESS_JWKS_PATH, + ESS_CONFIG_PATH, + ESS_FILES_PATH, +} from '../paths'; interface BaseOptions { tag?: string; @@ -158,7 +164,7 @@ const DEFAULT_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path', `${ESS_CONFIG_PATH}secrets/jwks.json`], - ['xpack.security.authc.realms.jwt.jwt1.claims.principal', 'sub'] + ['xpack.security.authc.realms.jwt.jwt1.claims.principal', 'sub'], ]; const DOCKER_SSL_ESARGS: Array<[string, string]> = [ @@ -467,7 +473,7 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles `${ESS_SECRETS_PATH}:${ESS_CONFIG_PATH}secrets/secrets.json:z`, '--volume', - `${ESS_JWKS_PATH}:${ESS_CONFIG_PATH}secrets/jwks.json:z`, + `${ESS_JWKS_PATH}:${ESS_CONFIG_PATH}secrets/jwks.json:z` ); } From b0b162620d092427b4875985c40dbdeb17a42e27 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 26 Aug 2023 05:14:54 +0100 Subject: [PATCH 123/182] refact(NA): change kibana_serverless_superuser to elastic_serverless --- packages/kbn-es/index.ts | 4 ++-- packages/kbn-es/src/ess_resources/README.md | 4 ++-- packages/kbn-es/src/ess_resources/operator_users.yml | 2 +- packages/kbn-es/src/ess_resources/users | 2 +- packages/kbn-es/src/ess_resources/users_roles | 2 +- packages/kbn-es/src/utils/ess_file_realm.ts | 4 ++-- packages/kbn-test/index.ts | 2 +- packages/kbn-test/src/kbn/index.ts | 2 +- packages/kbn-test/src/kbn/users.ts | 10 +++++----- .../observability/cypress/support/commands.ts | 2 +- x-pack/test_serverless/shared/config.base.ts | 4 ++-- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/kbn-es/index.ts b/packages/kbn-es/index.ts index 694e92e81ccb..3ccb220be6b5 100644 --- a/packages/kbn-es/index.ts +++ b/packages/kbn-es/index.ts @@ -10,7 +10,7 @@ export { run } from './src/cli'; export { Cluster } from './src/cluster'; export { SYSTEM_INDICES_SUPERUSER, - KIBANA_SERVERLESS_SUPERUSER, - KIBANA_SERVERLESS_SUPERUSER_PASSWORD, + ELASTIC_SERVERLESS_SUPERUSER, + ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, getDockerFileMountPath, } from './src/utils'; diff --git a/packages/kbn-es/src/ess_resources/README.md b/packages/kbn-es/src/ess_resources/README.md index f70f7a25ebb0..a7af386bcff1 100644 --- a/packages/kbn-es/src/ess_resources/README.md +++ b/packages/kbn-es/src/ess_resources/README.md @@ -8,13 +8,13 @@ The resources in this directory are used for seeding Elasticsearch Serverless (E The default superuser authentication to login to Kibana is: ``` -username: kibana_serverless_superuser +username: elastic_serverless password: changeme ``` ### Adding users -1. Add the user:encrypted_password to `users` file. The encrypted password for `kibana_serverless_superuser` is `changeme` if you want to reuse the value. +1. Add the user:encrypted_password to `users` file. The encrypted password for `elastic_serverless` is `changeme` if you want to reuse the value. 1. Set the new user's roles in `users_roles` file. 1. Add the username to `operator_users.yml` in the array for file realm users. diff --git a/packages/kbn-es/src/ess_resources/operator_users.yml b/packages/kbn-es/src/ess_resources/operator_users.yml index 63f6a6e99972..859226f258eb 100644 --- a/packages/kbn-es/src/ess_resources/operator_users.yml +++ b/packages/kbn-es/src/ess_resources/operator_users.yml @@ -1,5 +1,5 @@ operator: - - usernames: ["kibana_serverless_superuser", "system_indices_superuser"] + - usernames: ["elastic_serverless", "system_indices_superuser"] realm_type: "file" auth_type: "realm" - usernames: [ "elastic/kibana" ] diff --git a/packages/kbn-es/src/ess_resources/users b/packages/kbn-es/src/ess_resources/users index d982cc51aec7..add4b7325c23 100644 --- a/packages/kbn-es/src/ess_resources/users +++ b/packages/kbn-es/src/ess_resources/users @@ -1,2 +1,2 @@ -kibana_serverless_superuser:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW +elastic_serverless:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW system_indices_superuser:$2a$10$nN6sRtQl2KX9Gn8kV/.NpOLSk6Jwn8TehEDnZ7aaAgzyl/dy5PYzW diff --git a/packages/kbn-es/src/ess_resources/users_roles b/packages/kbn-es/src/ess_resources/users_roles index 622b8958772b..aa4204689860 100644 --- a/packages/kbn-es/src/ess_resources/users_roles +++ b/packages/kbn-es/src/ess_resources/users_roles @@ -1,2 +1,2 @@ -superuser:kibana_serverless_superuser +superuser:elastic_serverless system_indices_superuser:system_indices_superuser diff --git a/packages/kbn-es/src/utils/ess_file_realm.ts b/packages/kbn-es/src/utils/ess_file_realm.ts index cb7c019240e2..6b7745ac9351 100644 --- a/packages/kbn-es/src/utils/ess_file_realm.ts +++ b/packages/kbn-es/src/utils/ess_file_realm.ts @@ -6,5 +6,5 @@ * Side Public License, v 1. */ -export const KIBANA_SERVERLESS_SUPERUSER = 'kibana_serverless_superuser'; -export const KIBANA_SERVERLESS_SUPERUSER_PASSWORD = 'changeme'; +export const ELASTIC_SERVERLESS_SUPERUSER = 'elastic_serverless'; +export const ELASTIC_SERVERLESS_SUPERUSER_PASSWORD = 'changeme'; diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index 7f3a87ee45c6..5be415161a4a 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -38,7 +38,7 @@ export { kibanaTestUser, adminTestUser, systemIndicesSuperuser, - kibanaServerlessSuperuser, + kibanaTestSuperuserServerless, } from './src/kbn'; // @internal diff --git a/packages/kbn-test/src/kbn/index.ts b/packages/kbn-test/src/kbn/index.ts index 0c0929e918f3..6f2c009ce737 100644 --- a/packages/kbn-test/src/kbn/index.ts +++ b/packages/kbn-test/src/kbn/index.ts @@ -11,5 +11,5 @@ export { kibanaServerTestUser, adminTestUser, systemIndicesSuperuser, - kibanaServerlessSuperuser, + kibanaTestSuperuserServerless, } from './users'; diff --git a/packages/kbn-test/src/kbn/users.ts b/packages/kbn-test/src/kbn/users.ts index de8a7937d86d..9a68a55beb6e 100644 --- a/packages/kbn-test/src/kbn/users.ts +++ b/packages/kbn-test/src/kbn/users.ts @@ -8,8 +8,8 @@ import { SYSTEM_INDICES_SUPERUSER, - KIBANA_SERVERLESS_SUPERUSER, - KIBANA_SERVERLESS_SUPERUSER_PASSWORD, + ELASTIC_SERVERLESS_SUPERUSER, + ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, } from '@kbn/es'; const env = process.env; @@ -37,7 +37,7 @@ export const systemIndicesSuperuser = { password: env.TEST_ES_PASS || 'changeme', }; -export const kibanaServerlessSuperuser = { - username: KIBANA_SERVERLESS_SUPERUSER, - password: KIBANA_SERVERLESS_SUPERUSER_PASSWORD, +export const kibanaTestSuperuserServerless = { + username: ELASTIC_SERVERLESS_SUPERUSER, + password: ELASTIC_SERVERLESS_SUPERUSER_PASSWORD, }; diff --git a/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts b/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts index 369d3bf1cc35..3d3516397bdf 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cypress/support/commands.ts @@ -10,7 +10,7 @@ import 'cypress-axe'; Cypress.Commands.add('loginAsElasticUser', (path?: string) => { cy.visit(path ?? '/', { auth: { - username: 'kibana_serverless_superuser', + username: 'elastic_serverless', password: 'changeme', }, }); diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 32d6cf20eb79..f26178f636c4 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -13,7 +13,7 @@ import { REPO_ROOT } from '@kbn/repo-info'; import { esTestConfig, kbnTestConfig, - kibanaServerlessSuperuser, + kibanaTestSuperuserServerless, getDockerFileMountPath, } from '@kbn/test'; import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils'; @@ -22,7 +22,7 @@ import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; export default async () => { const servers = { kibana: { - ...kbnTestConfig.getUrlParts(kibanaServerlessSuperuser), + ...kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless), protocol: 'https', certificateAuthorities: [Fs.readFileSync(CA_CERT_PATH)], }, From 13d5ad1446656e74d57b05b03ca7c2606c7d12a0 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 26 Aug 2023 15:18:38 +0200 Subject: [PATCH 124/182] support ssl --- .../scripts/run_cypress/parallel.ts | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index a5dda9712387..69d0dac1afc7 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -224,10 +224,7 @@ export const cli = () => { }, }, kbnTestServer: { - serverArgs: [ - `--server.port=${kibanaPort}`, - `--elasticsearch.hosts=http://localhost:${esPort}`, - ], + serverArgs: [`--server.port=${kibanaPort}`], }, }, (vars) => { @@ -242,9 +239,10 @@ export const cli = () => { vars.kbnTestServer.serverArgs, (value) => !( - value.includes('--elasticsearch.hosts=http://localhost:9220') || + value.includes('--elasticsearch.hosts=') || value.includes('--xpack.fleet.agents.fleet_server.hosts') || - value.includes('--xpack.fleet.agents.elasticsearch.host') + value.includes('--xpack.fleet.agents.elasticsearch.host') || + value.includes('--server.publicBaseUrl') ) ); @@ -294,7 +292,24 @@ export const cli = () => { if (vars.serverless) { log.info(`Serverless mode detected`); - if (configFromTestFile?.productTypes) { + vars.kbnTestServer.serverArgs.push( + `--elasticsearch.hosts=https://localhost:${esPort}`, + `--server.publicBaseUrl=https://localhost:${kibanaPort}` + ); + vars.esTestCluster.serverArgs.push( + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://host.docker.internal:${kibanaPort}`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://host.docker.internal:${kibanaPort}/logout`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://host.docker.internal:${kibanaPort}/api/security/saml/callback` + ); + } else { + vars.kbnTestServer.serverArgs.push( + `--elasticsearch.hosts=http://localhost:${esPort}`, + `--server.publicBaseUrl=http://localhost:${kibanaPort}` + ); + } + + if (configFromTestFile?.productTypes) { + if (vars.serverless) { vars.kbnTestServer.serverArgs.push( `--xpack.securitySolutionServerless.productTypes=${JSON.stringify([ ...configFromTestFile.productTypes, @@ -307,11 +322,11 @@ export const cli = () => { ...configFromTestFile.productTypes, ])}` ); + } else { + log.warning( + `'ftrConfig.productTypes' ignored. Value applies only when running kibana is serverless.\nFile: ${filePath}` + ); } - } else if (configFromTestFile?.productTypes) { - log.warning( - `'ftrConfig.productTypes' ignored. Value applies only when running kibana is serverless.\nFile: ${filePath}` - ); } return vars; @@ -356,20 +371,16 @@ ${JSON.stringify(config.getAll(), null, 2)} { retries: 2, forever: false } ); - await pRetry( - async () => - runKibanaServer({ - procs, - config, - installDir: options?.installDir, - extraKbnOpts: - options?.installDir || options?.ci || !isOpen - ? [] - : ['--dev', '--no-dev-config', '--no-dev-credentials'], - onEarlyExit, - }), - { retries: 2, forever: false } - ); + await runKibanaServer({ + procs, + config, + installDir: options?.installDir, + extraKbnOpts: + options?.installDir || options?.ci || !isOpen + ? [] + : ['--dev', '--no-dev-config', '--no-dev-credentials'], + onEarlyExit, + }); await providers.loadAll(); From 039b863ced835444d44289f6ec65eec765a33a33 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 26 Aug 2023 18:38:07 +0100 Subject: [PATCH 125/182] fix(NA): running cypress tests parallel on serverless --- .../scripts/run_cypress/parallel.ts | 53 ++++++++++++++++--- x-pack/test_serverless/shared/config.base.ts | 1 + 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index a5dda9712387..823dbda52730 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -139,6 +139,8 @@ export const cli = () => { }; const getKibanaPort = (): T | number => { + return 5690; + if (isOpen) { return 5620; } @@ -205,6 +207,8 @@ export const cli = () => { const fleetServerPort: number = getFleetServerPort(); const configFromTestFile = parseTestFileConfig(filePath); + debugger; + const config = await readConfigFile( log, EsVersion.getDefault(), @@ -223,14 +227,25 @@ export const cli = () => { port: fleetServerPort, }, }, - kbnTestServer: { - serverArgs: [ - `--server.port=${kibanaPort}`, - `--elasticsearch.hosts=http://localhost:${esPort}`, - ], - }, + // CAUTION: Do not override here kbnTestServer.serverArgs + // or important configs like ssl key and certificate will be lost. + // Please do it in the section bellow on extendedSettings + // + // kbnTestServer: { + // serverArgs: [ + // ... + // ], + // }, }, (vars) => { + // NOTE: extending server args here as settingOverrides above is removing some important SSL configs + // like key and certificate + vars.kbnTestServer.serverArgs.concat([ + `--server.port=${kibanaPort}`, + `--elasticsearch.hosts=http://localhost:${esPort}`, + `--server.publicBaseUrl=http://localhost:${kibanaPort}` + ]) + const hasFleetServerArgs = _.some( vars.kbnTestServer.serverArgs, (value) => @@ -243,11 +258,35 @@ export const cli = () => { (value) => !( value.includes('--elasticsearch.hosts=http://localhost:9220') || + value.includes('--elasticsearch.hosts=https://localhost:9220') || value.includes('--xpack.fleet.agents.fleet_server.hosts') || - value.includes('--xpack.fleet.agents.elasticsearch.host') + value.includes('--xpack.fleet.agents.elasticsearch.host') || + (value.includes('--server.port=5620') && !isOpen) || + (value.includes('--server.publicBaseUrl=http://localhost:5620') && !isOpen) || + (value.includes('--server.publicBaseUrl=https://localhost:5620') && !isOpen) ) ); + // apply right protocol on hosts + vars.kbnTestServer.serverArgs = _.map(vars.kbnTestServer.serverArgs, (value) => { + if ( + vars.servers.elasticsearch.protocol === 'https' && + value.includes('--elasticsearch.hosts=http') + ) { + return value.replace('http', 'https'); + } + + if ( + vars.servers.kibana.protocol === 'https' && + (value.includes('--elasticsearch.hosts=http') || + value.includes('--server.publicBaseUrl=http')) + ) { + return value.replace('http', 'https'); + } + + return value; + }); + if ( configFromTestFile?.enableExperimental?.length && _.some(vars.kbnTestServer.serverArgs, (value) => diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index f26178f636c4..ed74fcb52a3d 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -75,6 +75,7 @@ export default async () => { `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://localhost:${servers.kibana.port}/api/security/saml/callback`, 'xpack.security.authc.realms.saml.cloud-saml-kibana.attributes.principal=urn:oid:0.0.7', ], + ssl: true // not needed as for serverless ssl is always on but added it anyway }, kbnTestServer: { From b99c4e7eaa7d79526e5ce2b7f0fe897bd74220cb Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 26 Aug 2023 18:47:29 +0100 Subject: [PATCH 126/182] chore(NA): remove debugger statement --- .../plugins/security_solution/scripts/run_cypress/parallel.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 823dbda52730..d729223d2022 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -207,8 +207,6 @@ export const cli = () => { const fleetServerPort: number = getFleetServerPort(); const configFromTestFile = parseTestFileConfig(filePath); - debugger; - const config = await readConfigFile( log, EsVersion.getDefault(), From e9850552e9547f97edbda13c04b25bf0a2e24475 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 26 Aug 2023 18:55:02 +0100 Subject: [PATCH 127/182] fix(NA): remove hardcoded port --- .../plugins/security_solution/scripts/run_cypress/parallel.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index d729223d2022..b21aa92d3e89 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -139,8 +139,6 @@ export const cli = () => { }; const getKibanaPort = (): T | number => { - return 5690; - if (isOpen) { return 5620; } From 45e2e9e2b894491f9d6fd5d725e98bef74fc9b11 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 26 Aug 2023 17:59:19 +0000 Subject: [PATCH 128/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../plugins/security_solution/scripts/run_cypress/parallel.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index b21aa92d3e89..b85907220d75 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -239,8 +239,8 @@ export const cli = () => { vars.kbnTestServer.serverArgs.concat([ `--server.port=${kibanaPort}`, `--elasticsearch.hosts=http://localhost:${esPort}`, - `--server.publicBaseUrl=http://localhost:${kibanaPort}` - ]) + `--server.publicBaseUrl=http://localhost:${kibanaPort}`, + ]); const hasFleetServerArgs = _.some( vars.kbnTestServer.serverArgs, From 9752249953a35a43136dd8857fa873f50a9e9ac0 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 26 Aug 2023 18:31:15 +0000 Subject: [PATCH 129/182] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- x-pack/test_serverless/shared/config.base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index ed74fcb52a3d..eb1390544203 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -75,7 +75,7 @@ export default async () => { `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://localhost:${servers.kibana.port}/api/security/saml/callback`, 'xpack.security.authc.realms.saml.cloud-saml-kibana.attributes.principal=urn:oid:0.0.7', ], - ssl: true // not needed as for serverless ssl is always on but added it anyway + ssl: true, // not needed as for serverless ssl is always on but added it anyway }, kbnTestServer: { From da17dc2e2d71c1a5c491f426b1200965796d63f9 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 27 Aug 2023 20:29:12 +0200 Subject: [PATCH 130/182] add ca certificate to esClient and kbnClient for es_archiver --- .../scripts/run_cypress/get_ftr_config.ts | 195 ++++++++++++++++++ .../scripts/run_cypress/parallel.ts | 191 ++--------------- .../cypress/support/es_archiver.ts | 18 +- 3 files changed, 225 insertions(+), 179 deletions(-) create mode 100644 x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts new file mode 100644 index 000000000000..06784efc496e --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import _ from 'lodash'; + +import { EsVersion, readConfigFile } from '@kbn/test'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { getLocalhostRealIp } from '../endpoint/common/localhost_services'; +import type { parseTestFileConfig } from './utils'; + +export const getFTRConfig = ({ + log, + esPort, + kibanaPort, + fleetServerPort, + ftrConfigFilePath, + specFilePath, + specFileFTRConfig, + isOpen, +}: { + log: ToolingLog; + esPort: number; + kibanaPort: number; + fleetServerPort: number; + ftrConfigFilePath: string; + specFilePath: string; + specFileFTRConfig: ReturnType; + isOpen: boolean; +}) => + readConfigFile( + log, + EsVersion.getDefault(), + ftrConfigFilePath, + { + servers: { + elasticsearch: { + port: esPort, + }, + kibana: { + port: kibanaPort, + }, + fleetserver: { + port: fleetServerPort, + }, + }, + // CAUTION: Do not override here kbnTestServer.serverArgs + // or important configs like ssl key and certificate will be lost. + // Please do it in the section bellow on extendedSettings + // + // kbnTestServer: { + // serverArgs: [ + // ... + // ], + // }, + }, + (vars) => { + const hostRealIp = getLocalhostRealIp(); + + // NOTE: extending server args here as settingOverrides above is removing some important SSL configs + // like key and certificate + vars.kbnTestServer.serverArgs.concat([ + `--server.port=${kibanaPort}`, + `--elasticsearch.hosts=http://localhost:${esPort}`, + `--server.publicBaseUrl=http://localhost:${kibanaPort}`, + ]); + + const hasFleetServerArgs = _.some( + vars.kbnTestServer.serverArgs, + (value) => + value.includes('--xpack.fleet.agents.fleet_server.hosts') || + value.includes('--xpack.fleet.agents.elasticsearch.host') + ); + + vars.kbnTestServer.serverArgs = _.filter( + vars.kbnTestServer.serverArgs, + (value) => + !( + value.includes('--elasticsearch.hosts=http://localhost:9220') || + value.includes('--elasticsearch.hosts=https://localhost:9220') || + value.includes('--xpack.fleet.agents.fleet_server.hosts') || + value.includes('--xpack.fleet.agents.elasticsearch.host') || + (value.includes('--server.port=5620') && !isOpen) || + (value.includes('--server.publicBaseUrl=http://localhost:5620') && !isOpen) || + (value.includes('--server.publicBaseUrl=https://localhost:5620') && !isOpen) + ) + ); + + // apply right protocol on hosts + vars.kbnTestServer.serverArgs = _.map(vars.kbnTestServer.serverArgs, (value) => { + if ( + vars.servers.elasticsearch.protocol === 'https' && + value.includes('--elasticsearch.hosts=http') + ) { + return value.replace('http', 'https'); + } + + if ( + vars.servers.kibana.protocol === 'https' && + (value.includes('--elasticsearch.hosts=http') || + value.includes('--server.publicBaseUrl=http')) + ) { + return value.replace('http', 'https'); + } + + return value; + }); + + if ( + specFileFTRConfig?.enableExperimental?.length && + _.some(vars.kbnTestServer.serverArgs, (value) => + value.includes('--xpack.securitySolution.enableExperimental') + ) + ) { + vars.kbnTestServer.serverArgs = _.filter( + vars.kbnTestServer.serverArgs, + (value) => !value.includes('--xpack.securitySolution.enableExperimental') + ); + vars.kbnTestServer.serverArgs.push( + `--xpack.securitySolution.enableExperimental=${JSON.stringify( + specFileFTRConfig?.enableExperimental + )}` + ); + } + + if (specFileFTRConfig?.license) { + if (vars.serverless) { + log.warning( + `'ftrConfig.license' ignored. Value does not apply to kibana when running in serverless.\nFile: ${specFilePath}` + ); + } else { + vars.esTestCluster.license = specFileFTRConfig.license; + } + } + + if (hasFleetServerArgs) { + vars.kbnTestServer.serverArgs.push( + `--xpack.fleet.agents.fleet_server.hosts=["https://${hostRealIp}:${fleetServerPort}"]` + ); + vars.kbnTestServer.serverArgs.push( + `--xpack.fleet.agents.elasticsearch.host=http://${hostRealIp}:${esPort}` + ); + + if (vars.serverless) { + vars.kbnTestServer.serverArgs.push(`--xpack.fleet.internal.fleetServerStandalone=false`); + } + } + + // Serverless Specific + if (vars.serverless) { + log.info(`Serverless mode detected`); + + vars.kbnTestServer.serverArgs.push( + `--elasticsearch.hosts=https://localhost:${esPort}`, + `--server.publicBaseUrl=https://localhost:${kibanaPort}` + ); + vars.esTestCluster.serverArgs.push( + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://host.docker.internal:${kibanaPort}`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://host.docker.internal:${kibanaPort}/logout`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://host.docker.internal:${kibanaPort}/api/security/saml/callback` + ); + } else { + vars.kbnTestServer.serverArgs.push( + `--elasticsearch.hosts=http://localhost:${esPort}`, + `--server.publicBaseUrl=http://localhost:${kibanaPort}` + ); + } + + if (specFileFTRConfig?.productTypes) { + if (vars.serverless) { + vars.kbnTestServer.serverArgs.push( + `--xpack.securitySolutionServerless.productTypes=${JSON.stringify([ + ...specFileFTRConfig.productTypes, + // Why spread it twice? + // The `serverless.security.yml` file by default includes two product types as of this change. + // Because it's an array, we need to ensure that existing values are "removed" and the ones + // defined here are added. To do that, we duplicate the `productTypes` passed so that all array + // elements in that YAML file are updated. The Security serverless plugin has code in place to + // dedupe. + ...specFileFTRConfig.productTypes, + ])}` + ); + } else { + log.warning( + `'ftrConfig.productTypes' ignored. Value applies only when running kibana is serverless.\nFile: ${specFilePath}` + ); + } + } + + return vars; + } + ); diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 9eb1d489c97a..3f228f188a26 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -18,13 +18,7 @@ import minimatch from 'minimatch'; import path from 'path'; import grep from '@cypress/grep/src/plugin'; -import { - EsVersion, - FunctionalTestRunner, - readConfigFile, - runElasticsearch, - runKibanaServer, -} from '@kbn/test'; +import { EsVersion, FunctionalTestRunner, runElasticsearch, runKibanaServer } from '@kbn/test'; import { Lifecycle, @@ -35,8 +29,8 @@ import { import { createFailError } from '@kbn/dev-cli-errors'; import pRetry from 'p-retry'; import { renderSummaryTable } from './print_run'; -import { getLocalhostRealIp } from '../endpoint/common/localhost_services'; import { isSkipped, parseTestFileConfig } from './utils'; +import { getFTRConfig } from './get_ftr_config'; /** * Retrieve test files using a glob pattern. @@ -183,8 +177,6 @@ export const cli = () => { writeTo: process.stdout, }); - const hostRealIp = getLocalhostRealIp(); - await pMap( files, async (filePath) => { @@ -203,172 +195,21 @@ export const cli = () => { const esPort: number = getEsPort(); const kibanaPort: number = getKibanaPort(); const fleetServerPort: number = getFleetServerPort(); - const configFromTestFile = parseTestFileConfig(filePath); + const specFileFTRConfig = parseTestFileConfig(filePath); + const ftrConfigFilePath = path.resolve( + _.isArray(argv.ftrConfigFile) ? _.last(argv.ftrConfigFile) : argv.ftrConfigFile + ); - const config = await readConfigFile( + const config = await getFTRConfig({ log, - EsVersion.getDefault(), - path.resolve( - _.isArray(argv.ftrConfigFile) ? _.last(argv.ftrConfigFile) : argv.ftrConfigFile - ), - { - servers: { - elasticsearch: { - port: esPort, - }, - kibana: { - port: kibanaPort, - }, - fleetserver: { - port: fleetServerPort, - }, - }, - // CAUTION: Do not override here kbnTestServer.serverArgs - // or important configs like ssl key and certificate will be lost. - // Please do it in the section bellow on extendedSettings - // - // kbnTestServer: { - // serverArgs: [ - // ... - // ], - // }, - }, - (vars) => { - // NOTE: extending server args here as settingOverrides above is removing some important SSL configs - // like key and certificate - vars.kbnTestServer.serverArgs.concat([ - `--server.port=${kibanaPort}`, - `--elasticsearch.hosts=http://localhost:${esPort}`, - `--server.publicBaseUrl=http://localhost:${kibanaPort}`, - ]); - - const hasFleetServerArgs = _.some( - vars.kbnTestServer.serverArgs, - (value) => - value.includes('--xpack.fleet.agents.fleet_server.hosts') || - value.includes('--xpack.fleet.agents.elasticsearch.host') - ); - - vars.kbnTestServer.serverArgs = _.filter( - vars.kbnTestServer.serverArgs, - (value) => - !( - value.includes('--elasticsearch.hosts=http://localhost:9220') || - value.includes('--elasticsearch.hosts=https://localhost:9220') || - value.includes('--xpack.fleet.agents.fleet_server.hosts') || - value.includes('--xpack.fleet.agents.elasticsearch.host') || - (value.includes('--server.port=5620') && !isOpen) || - (value.includes('--server.publicBaseUrl=http://localhost:5620') && !isOpen) || - (value.includes('--server.publicBaseUrl=https://localhost:5620') && !isOpen) - ) - ); - - // apply right protocol on hosts - vars.kbnTestServer.serverArgs = _.map(vars.kbnTestServer.serverArgs, (value) => { - if ( - vars.servers.elasticsearch.protocol === 'https' && - value.includes('--elasticsearch.hosts=http') - ) { - return value.replace('http', 'https'); - } - - if ( - vars.servers.kibana.protocol === 'https' && - (value.includes('--elasticsearch.hosts=http') || - value.includes('--server.publicBaseUrl=http')) - ) { - return value.replace('http', 'https'); - } - - return value; - }); - - if ( - configFromTestFile?.enableExperimental?.length && - _.some(vars.kbnTestServer.serverArgs, (value) => - value.includes('--xpack.securitySolution.enableExperimental') - ) - ) { - vars.kbnTestServer.serverArgs = _.filter( - vars.kbnTestServer.serverArgs, - (value) => !value.includes('--xpack.securitySolution.enableExperimental') - ); - vars.kbnTestServer.serverArgs.push( - `--xpack.securitySolution.enableExperimental=${JSON.stringify( - configFromTestFile?.enableExperimental - )}` - ); - } - - if (configFromTestFile?.license) { - if (vars.serverless) { - log.warning( - `'ftrConfig.license' ignored. Value does not apply to kibana when running in serverless.\nFile: ${filePath}` - ); - } else { - vars.esTestCluster.license = configFromTestFile.license; - } - } - - if (hasFleetServerArgs) { - vars.kbnTestServer.serverArgs.push( - `--xpack.fleet.agents.fleet_server.hosts=["https://${hostRealIp}:${fleetServerPort}"]` - ); - vars.kbnTestServer.serverArgs.push( - `--xpack.fleet.agents.elasticsearch.host=http://${hostRealIp}:${esPort}` - ); - - if (vars.serverless) { - vars.kbnTestServer.serverArgs.push( - `--xpack.fleet.internal.fleetServerStandalone=false` - ); - } - } - - // Serverless Specific - if (vars.serverless) { - log.info(`Serverless mode detected`); - - vars.kbnTestServer.serverArgs.push( - `--elasticsearch.hosts=https://localhost:${esPort}`, - `--server.publicBaseUrl=https://localhost:${kibanaPort}` - ); - vars.esTestCluster.serverArgs.push( - `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://host.docker.internal:${kibanaPort}`, - `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://host.docker.internal:${kibanaPort}/logout`, - `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://host.docker.internal:${kibanaPort}/api/security/saml/callback` - ); - } else { - vars.kbnTestServer.serverArgs.push( - `--elasticsearch.hosts=http://localhost:${esPort}`, - `--server.publicBaseUrl=http://localhost:${kibanaPort}` - ); - } - - if (configFromTestFile?.productTypes) { - if (vars.serverless) { - vars.kbnTestServer.serverArgs.push( - `--xpack.securitySolutionServerless.productTypes=${JSON.stringify([ - ...configFromTestFile.productTypes, - // Why spread it twice? - // The `serverless.security.yml` file by default includes two product types as of this change. - // Because it's an array, we need to ensure that existing values are "removed" and the ones - // defined here are added. To do that, we duplicate the `productTypes` passed so that all array - // elements in that YAML file are updated. The Security serverless plugin has code in place to - // dedupe. - ...configFromTestFile.productTypes, - ])}` - ); - } else { - log.warning( - `'ftrConfig.productTypes' ignored. Value applies only when running kibana is serverless.\nFile: ${filePath}` - ); - } - } - - return vars; - } - ); + esPort, + kibanaPort, + fleetServerPort, + ftrConfigFilePath, + specFilePath: filePath, + specFileFTRConfig, + isOpen, + }); log.info(` ---------------------------------------------- @@ -486,6 +327,8 @@ ${JSON.stringify(config.getAll(), null, 2)} KIBANA_USERNAME: config.get('servers.kibana.username'), KIBANA_PASSWORD: config.get('servers.kibana.password'), + IS_SERVERLESS: config.get('serverless'), + ...argv.env, }; diff --git a/x-pack/test/security_solution_cypress/cypress/support/es_archiver.ts b/x-pack/test/security_solution_cypress/cypress/support/es_archiver.ts index 42ddb4a52638..7b283702ae12 100644 --- a/x-pack/test/security_solution_cypress/cypress/support/es_archiver.ts +++ b/x-pack/test/security_solution_cypress/cypress/support/es_archiver.ts @@ -5,10 +5,12 @@ * 2.0. */ +import Fs from 'fs'; +import * as Url from 'url'; import { EsArchiver } from '@kbn/es-archiver'; -import { KbnClient } from '@kbn/test'; -import { Client, HttpConnection } from '@elastic/elasticsearch'; +import { createEsClientForTesting, KbnClient, systemIndicesSuperuser } from '@kbn/test'; import { ToolingLog } from '@kbn/tooling-log'; +import { CA_CERT_PATH } from '@kbn/dev-utils'; export const esArchiver = ( on: Cypress.PluginEvents, @@ -16,14 +18,20 @@ export const esArchiver = ( ): EsArchiver => { const log = new ToolingLog({ level: 'verbose', writeTo: process.stdout }); - const client = new Client({ - node: config.env.ELASTICSEARCH_URL, - Connection: HttpConnection, + const isServerless = config.env.IS_SERVERLESS; + + const client = createEsClientForTesting({ + esUrl: Url.format(config.env.ELASTICSEARCH_URL), + // Use system indices user so tests can write to system indices + authOverride: !isServerless ? systemIndicesSuperuser : undefined, }); const kbnClient = new KbnClient({ log, url: config.env.CYPRESS_BASE_URL as string, + ...(config.env.ELASTICSEARCH_URL.includes('https') + ? { certificateAuthorities: [Fs.readFileSync(CA_CERT_PATH)] } + : {}), }); const esArchiverInstance = new EsArchiver({ From 214e69decccfc0177f4638abde5538e7952e6021 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sun, 27 Aug 2023 18:34:43 +0000 Subject: [PATCH 131/182] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test/security_solution_cypress/cypress/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_cypress/cypress/tsconfig.json b/x-pack/test/security_solution_cypress/cypress/tsconfig.json index 8ee21e233a22..ae3bb49693d7 100644 --- a/x-pack/test/security_solution_cypress/cypress/tsconfig.json +++ b/x-pack/test/security_solution_cypress/cypress/tsconfig.json @@ -43,5 +43,6 @@ "@kbn/fleet-plugin", "@kbn/cases-components", "@kbn/security-solution-plugin", + "@kbn/dev-utils", ] } From 6bf366ad037a994b343cf8b3a913a936a476a370 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 27 Aug 2023 20:45:09 +0200 Subject: [PATCH 132/182] fix --- .../scripts/run_cypress/get_ftr_config.ts | 22 ++++++++----------- .../cypress/e2e/data_sources/sourcerer.cy.ts | 3 ++- .../cypress/e2e/explore/cases/creation.cy.ts | 2 +- .../e2e/explore/cases/privileges.cy.ts | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts index 06784efc496e..43832c303f7d 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts @@ -60,14 +60,6 @@ export const getFTRConfig = ({ (vars) => { const hostRealIp = getLocalhostRealIp(); - // NOTE: extending server args here as settingOverrides above is removing some important SSL configs - // like key and certificate - vars.kbnTestServer.serverArgs.concat([ - `--server.port=${kibanaPort}`, - `--elasticsearch.hosts=http://localhost:${esPort}`, - `--server.publicBaseUrl=http://localhost:${kibanaPort}`, - ]); - const hasFleetServerArgs = _.some( vars.kbnTestServer.serverArgs, (value) => @@ -79,16 +71,20 @@ export const getFTRConfig = ({ vars.kbnTestServer.serverArgs, (value) => !( - value.includes('--elasticsearch.hosts=http://localhost:9220') || - value.includes('--elasticsearch.hosts=https://localhost:9220') || + value.includes('--elasticsearch.hosts') || value.includes('--xpack.fleet.agents.fleet_server.hosts') || value.includes('--xpack.fleet.agents.elasticsearch.host') || - (value.includes('--server.port=5620') && !isOpen) || - (value.includes('--server.publicBaseUrl=http://localhost:5620') && !isOpen) || - (value.includes('--server.publicBaseUrl=https://localhost:5620') && !isOpen) + value.includes('--server.port') ) ); + // NOTE: extending server args here as settingOverrides above is removing some important SSL configs + // like key and certificate + vars.kbnTestServer.serverArgs.push([ + `--server.port=${kibanaPort}`, + `--elasticsearch.hosts=http://localhost:${esPort}`, + ]); + // apply right protocol on hosts vars.kbnTestServer.serverArgs = _.map(vars.kbnTestServer.serverArgs, (value) => { if ( diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts index 1a776101e4f8..ce6b6454c412 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts @@ -50,7 +50,8 @@ describe('Sourcerer', () => { cy.task('esArchiverResetKibana'); dataViews.forEach((dataView: string) => postDataView(dataView)); }); - describe('permissions', { tags: '@ess' }, () => { + + describe('permissions', { tags: ['@ess', '@brokenInServerless'] }, () => { before(() => { createUsersAndRoles(usersToCreate, rolesToCreate); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts index ee654719a439..4de348c5bbb4 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/creation.cy.ts @@ -53,7 +53,7 @@ import { loginWithUser, visit, visitWithoutDateRange } from '../../../tasks/logi import { CASES_URL, OVERVIEW_URL } from '../../../urls/navigation'; -describe('Cases', { tags: ['@ess', '@serverless'] }, () => { +describe('Cases', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); createTimeline(getCase1().timeline).then((response) => diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/privileges.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/privileges.cy.ts index 57000344f119..e4d35947d142 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/privileges.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/cases/privileges.cy.ts @@ -48,7 +48,7 @@ const testCase: TestCaseWithoutTimeline = { owner: 'securitySolution', }; -describe('Cases privileges', { tags: '@ess' }, () => { +describe('Cases privileges', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); createUsersAndRoles(usersToCreate, rolesToCreate); From c67ef87d1e07f5602cb3bb9f3263ab0a0e9d7ac9 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 27 Aug 2023 21:21:50 +0200 Subject: [PATCH 133/182] fix --- .../security_solution/scripts/run_cypress/get_ftr_config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts index 43832c303f7d..baa79ca99edb 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts @@ -80,10 +80,10 @@ export const getFTRConfig = ({ // NOTE: extending server args here as settingOverrides above is removing some important SSL configs // like key and certificate - vars.kbnTestServer.serverArgs.push([ + vars.kbnTestServer.serverArgs.push( `--server.port=${kibanaPort}`, `--elasticsearch.hosts=http://localhost:${esPort}`, - ]); + ); // apply right protocol on hosts vars.kbnTestServer.serverArgs = _.map(vars.kbnTestServer.serverArgs, (value) => { From 500cf7b3863cdc1cbc795ede22de4c58e0861054 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sun, 27 Aug 2023 19:26:12 +0000 Subject: [PATCH 134/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../security_solution/scripts/run_cypress/get_ftr_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts index baa79ca99edb..11f39f6b4a27 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/get_ftr_config.ts @@ -82,7 +82,7 @@ export const getFTRConfig = ({ // like key and certificate vars.kbnTestServer.serverArgs.push( `--server.port=${kibanaPort}`, - `--elasticsearch.hosts=http://localhost:${esPort}`, + `--elasticsearch.hosts=http://localhost:${esPort}` ); // apply right protocol on hosts From 89a050f7b3062a92e7b75bf7e6963a2f691fce06 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 28 Aug 2023 08:39:14 +0200 Subject: [PATCH 135/182] skip broken tests --- .buildkite/pipelines/pull_request/base.yml | 27 +++---- .../pull_request/osquery_cypress.yml | 27 +++---- packages/kbn-es/src/utils/docker.test.ts | 16 +++- .../e2e/detection_alerts/alerts_charts.cy.ts | 76 ++++++++++--------- .../detection_alerts/cti_enrichments.cy.ts | 3 +- .../dashboards/enable_risk_score.cy.ts | 4 +- .../dashboards/upgrade_risk_score.cy.ts | 54 +++++++------ .../e2e/explore/host_details/risk_tab.cy.ts | 3 +- .../e2e/explore/hosts/hosts_risk_column.cy.ts | 3 +- .../cypress/e2e/inspect/inspect_button.cy.ts | 3 +- .../alert_details_right_panel_json_tab.cy.ts | 3 +- .../cypress/e2e/overview/cti_link_panel.cy.ts | 3 +- .../test_suites/common/data_view_mgmt.ts | 3 +- 13 files changed, 129 insertions(+), 96 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index bb96479a2b83..ed0b4f6e1886 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -110,19 +110,20 @@ steps: artifact_paths: - "target/kibana-security-solution/**/*" - - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh - label: 'Serverless Security Defend Workflows Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 40 - soft_fail: true - retry: - automatic: - - exit_status: '*' - limit: 1 - artifact_paths: - - "target/kibana-security-solution/**/*" + # status_exception: Native role management is not enabled in this Elasticsearch instance + # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh + # label: 'Serverless Security Defend Workflows Cypress Tests' + # agents: + # queue: n2-4-spot + # depends_on: build + # timeout_in_minutes: 40 + # soft_fail: true + # retry: + # automatic: + # - exit_status: '*' + # limit: 1 + # artifact_paths: + # - "target/kibana-security-solution/**/*" - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh label: 'Serverless Security Investigations Cypress Tests' diff --git a/.buildkite/pipelines/pull_request/osquery_cypress.yml b/.buildkite/pipelines/pull_request/osquery_cypress.yml index 07e26e8f1ff6..c56d94524f60 100644 --- a/.buildkite/pipelines/pull_request/osquery_cypress.yml +++ b/.buildkite/pipelines/pull_request/osquery_cypress.yml @@ -25,16 +25,17 @@ steps: artifact_paths: - "target/kibana-osquery/**/*" - - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh - label: 'Serverless Osquery Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 50 - parallelism: 6 - retry: - automatic: - - exit_status: '*' - limit: 1 - artifact_paths: - - "target/kibana-osquery/**/*" + # Error: self-signed certificate in certificate chain + # - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh + # label: 'Serverless Osquery Cypress Tests' + # agents: + # queue: n2-4-spot + # depends_on: build + # timeout_in_minutes: 50 + # parallelism: 6 + # retry: + # automatic: + # - exit_status: '*' + # limit: 1 + # artifact_paths: + # - "target/kibana-osquery/**/*" diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index e49736ce9ae1..510619976c41 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -341,7 +341,7 @@ describe('resolveEsArgs()', () => { ssl: true, }); - expect(esArgs).toHaveLength(20); + expect(esArgs).toHaveLength(32); expect(esArgs).not.toEqual(expect.arrayContaining(['xpack.security.enabled=false'])); expect(esArgs).toMatchInlineSnapshot(` Array [ @@ -365,6 +365,18 @@ describe('resolveEsArgs()', () => { "xpack.security.transport.ssl.verification_mode=certificate", "--env", "xpack.security.operator_privileges.enabled=true", + "--env", + "xpack.security.authc.realms.jwt.jwt1.client_authentication.type=shared_secret", + "--env", + "xpack.security.authc.realms.jwt.jwt1.order=-98", + "--env", + "xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/", + "--env", + "xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch", + "--env", + "xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=/usr/share/elasticsearch/config/secrets/jwks.json", + "--env", + "xpack.security.authc.realms.jwt.jwt1.claims.principal=sub", ] `); }); @@ -418,7 +430,7 @@ describe('setupServerlessVolumes()', () => { (path) => !volumeCmd.some((cmd) => cmd.includes(path)) ); - expect(volumeCmd).toHaveLength(18); + expect(volumeCmd).toHaveLength(20); expect(pathsNotIncludedInCmd).toEqual([]); }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alerts_charts.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alerts_charts.cy.ts index 656239c7308d..313d2a625ad9 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alerts_charts.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/alerts_charts.cy.ts @@ -24,46 +24,50 @@ import { } from '../../screens/search_bar'; import { TOASTER } from '../../screens/alerts_detection_rules'; -describe('Histogram legend hover actions', { tags: ['@ess', '@serverless'] }, () => { - const ruleConfigs = getNewRule(); +describe( + 'Histogram legend hover actions', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + const ruleConfigs = getNewRule(); - before(() => { - cleanKibana(); - }); + before(() => { + cleanKibana(); + }); - beforeEach(() => { - login(); - createRule(getNewRule({ rule_id: 'new custom rule' })); - visit(ALERTS_URL); - selectAlertsHistogram(); - }); + beforeEach(() => { + login(); + createRule(getNewRule({ rule_id: 'new custom rule' })); + visit(ALERTS_URL); + selectAlertsHistogram(); + }); - it('Filter in/out should add a filter to KQL bar', function () { - const expectedNumberOfAlerts = 2; - clickAlertsHistogramLegend(); - clickAlertsHistogramLegendFilterFor(ruleConfigs.name); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( - 'have.text', - `kibana.alert.rule.name: ${ruleConfigs.name}` - ); - cy.get(ALERTS_COUNT).should('have.text', `${expectedNumberOfAlerts} alerts`); + it('Filter in/out should add a filter to KQL bar', function () { + const expectedNumberOfAlerts = 2; + clickAlertsHistogramLegend(); + clickAlertsHistogramLegendFilterFor(ruleConfigs.name); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( + 'have.text', + `kibana.alert.rule.name: ${ruleConfigs.name}` + ); + cy.get(ALERTS_COUNT).should('have.text', `${expectedNumberOfAlerts} alerts`); - clickAlertsHistogramLegend(); - clickAlertsHistogramLegendFilterOut(ruleConfigs.name); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( - 'have.text', - `NOT kibana.alert.rule.name: ${ruleConfigs.name}` - ); - cy.get(ALERTS_COUNT).should('not.exist'); + clickAlertsHistogramLegend(); + clickAlertsHistogramLegendFilterOut(ruleConfigs.name); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should( + 'have.text', + `NOT kibana.alert.rule.name: ${ruleConfigs.name}` + ); + cy.get(ALERTS_COUNT).should('not.exist'); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM_DELETE).click(); - cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should('not.exist'); - }); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM_DELETE).click(); + cy.get(GLOBAL_SEARCH_BAR_FILTER_ITEM).should('not.exist'); + }); - it('Add To Timeline', function () { - clickAlertsHistogramLegend(); - clickAlertsHistogramLegendAddToTimeline(ruleConfigs.name); + it('Add To Timeline', function () { + clickAlertsHistogramLegend(); + clickAlertsHistogramLegendAddToTimeline(ruleConfigs.name); - cy.get(TOASTER).should('have.text', `Added ${ruleConfigs.name} to timeline`); - }); -}); + cy.get(TOASTER).should('have.text', `Added ${ruleConfigs.name} to timeline`); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/cti_enrichments.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/cti_enrichments.cy.ts index 27921bb9b2d7..bea55b3e62f5 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/cti_enrichments.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/cti_enrichments.cy.ts @@ -28,9 +28,10 @@ import { openJsonView, openThreatIndicatorDetails } from '../../tasks/alerts_det import { DETECTIONS_RULE_MANAGEMENT_URL } from '../../urls/navigation'; import { addsFieldsToTimeline } from '../../tasks/rule_details'; -describe('CTI Enrichment', { tags: ['@ess', '@serverless'] }, () => { +describe('CTI Enrichment', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); + // illegal_argument_exception: unknown setting [index.lifecycle.rollover_alias] cy.task('esArchiverLoad', 'threat_indicator'); cy.task('esArchiverLoad', 'suspicious_source_event'); login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts index e509125fb7ab..b0ce08a310e6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts @@ -55,7 +55,7 @@ describe('Enable risk scores', { tags: ['@ess', '@serverless'] }, () => { cy.get(ENABLE_HOST_RISK_SCORE_BUTTON).should('exist'); }); - it('should install host risk score successfully', () => { + it('should install host risk score successfully', { tags: ['@brokenInServerless']} () => { interceptInstallRiskScoreModule(); clickEnableRiskScore(RiskScoreEntity.host); waitForInstallRiskScoreModule(); @@ -89,7 +89,7 @@ describe('Enable risk scores', { tags: ['@ess', '@serverless'] }, () => { cy.get(ENABLE_USER_RISK_SCORE_BUTTON).should('exist'); }); - it('should install user risk score successfully', () => { + it('should install user risk score successfully', { tags: ['@brokenInServerless']} () => { interceptInstallRiskScoreModule(); clickEnableRiskScore(RiskScoreEntity.user); waitForInstallRiskScoreModule(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/upgrade_risk_score.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/upgrade_risk_score.cy.ts index 0d327b36f517..fcb33ff4de23 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/upgrade_risk_score.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/upgrade_risk_score.cy.ts @@ -59,31 +59,39 @@ describe('Upgrade risk scores', { tags: ['@ess', '@serverless'] }, () => { cy.get(UPGRADE_USER_RISK_SCORE_BUTTON).should('be.visible'); }); - it('should show a confirmation modal for upgrading host risk score and display a link to host risk score Elastic doc', () => { - clickUpgradeRiskScore(RiskScoreEntity.host); - cy.get(UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.host)).should('exist'); - - cy.get(UPGRADE_CANCELLATION_BUTTON) - .get(`${UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.host)} a`) - .then((link) => { - expect(link.prop('href')).to.eql( - `https://www.elastic.co/guide/en/security/current/${RiskScoreEntity.host}-risk-score.html` - ); - }); - }); + it( + 'should show a confirmation modal for upgrading host risk score and display a link to host risk score Elastic doc', + { tags: ['@brokenInServerless'] }, + () => { + clickUpgradeRiskScore(RiskScoreEntity.host); + cy.get(UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.host)).should('exist'); - it('should show a confirmation modal for upgrading user risk score and display a link to user risk score Elastic doc', () => { - clickUpgradeRiskScore(RiskScoreEntity.user); - cy.get(UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.user)).should('exist'); + cy.get(UPGRADE_CANCELLATION_BUTTON) + .get(`${UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.host)} a`) + .then((link) => { + expect(link.prop('href')).to.eql( + `https://www.elastic.co/guide/en/security/current/${RiskScoreEntity.host}-risk-score.html` + ); + }); + } + ); + + it( + 'should show a confirmation modal for upgrading user risk score and display a link to user risk score Elastic doc', + { tags: ['@brokenInServerless'] }, + () => { + clickUpgradeRiskScore(RiskScoreEntity.user); + cy.get(UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.user)).should('exist'); - cy.get(UPGRADE_CANCELLATION_BUTTON) - .get(`${UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.user)} a`) - .then((link) => { - expect(link.prop('href')).to.eql( - `https://www.elastic.co/guide/en/security/current/${RiskScoreEntity.user}-risk-score.html` - ); - }); - }); + cy.get(UPGRADE_CANCELLATION_BUTTON) + .get(`${UPGRADE_CONFIRMATION_MODAL(RiskScoreEntity.user)} a`) + .then((link) => { + expect(link.prop('href')).to.eql( + `https://www.elastic.co/guide/en/security/current/${RiskScoreEntity.user}-risk-score.html` + ); + }); + } + ); }); const versions: Array<'8.3' | '8.4'> = ['8.3', '8.4']; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/host_details/risk_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/host_details/risk_tab.cy.ts index cdfd01c034c3..25ad65dfa752 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/host_details/risk_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/host_details/risk_tab.cy.ts @@ -10,9 +10,10 @@ import { login, visitHostDetailsPage } from '../../../tasks/login'; import { cleanKibana, waitForTableToLoad } from '../../../tasks/common'; import { TABLE_CELL, TABLE_ROWS } from '../../../screens/alerts_details'; -describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { +describe('risk tab', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); + // illegal_argument_exception: unknown setting [index.lifecycle.rollover_alias] cy.task('esArchiverLoad', 'risk_hosts'); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/hosts_risk_column.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/hosts_risk_column.cy.ts index 664e5767dc2b..76d16a585820 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/hosts_risk_column.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/hosts_risk_column.cy.ts @@ -12,9 +12,10 @@ import { cleanKibana } from '../../../tasks/common'; import { TABLE_CELL } from '../../../screens/alerts_details'; import { kqlSearch } from '../../../tasks/security_header'; -describe('All hosts table', { tags: ['@ess', '@serverless'] }, () => { +describe('All hosts table', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { cleanKibana(); + // illegal_argument_exception: unknown setting [index.lifecycle.name] cy.task('esArchiverLoad', 'risk_hosts'); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/inspect/inspect_button.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/inspect/inspect_button.cy.ts index 209422bcf7eb..21d83651f6eb 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/inspect/inspect_button.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/inspect/inspect_button.cy.ts @@ -22,8 +22,9 @@ import { selectDataView } from '../../tasks/sourcerer'; const DATA_VIEW = 'auditbeat-*'; -describe('Inspect Explore pages', { tags: ['@ess', '@serverless'] }, () => { +describe('Inspect Explore pages', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { before(() => { + // illegal_argument_exception: unknown setting [index.lifecycle.name] cy.task('esArchiverLoad', 'risk_users'); cy.task('esArchiverLoad', 'risk_hosts'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts index 01c11a671cb6..c5726980196e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts @@ -16,7 +16,8 @@ import { getNewRule } from '../../../../objects/rule'; import { ALERTS_URL } from '../../../../urls/navigation'; import { waitForAlertsToPopulate } from '../../../../tasks/create_new_rule'; -describe( +// Failing constantly on CI +describe.skip( 'Alert details expandable flyout right panel json tab', { tags: ['@ess', '@brokenInServerless'] }, () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/overview/cti_link_panel.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/overview/cti_link_panel.cy.ts index fd41df836216..e3ac2971a16b 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/overview/cti_link_panel.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/overview/cti_link_panel.cy.ts @@ -30,8 +30,9 @@ describe('CTI Link Panel', { tags: ['@ess', '@serverless'] }, () => { .and('match', /app\/integrations\/browse\/threat_intel/); }); - describe('enabled threat intel module', () => { + describe('enabled threat intel module', { tags: ['@brokenInServerless'] }, () => { before(() => { + // illegal_argument_exception: unknown setting [index.lifecycle.name] cy.task('esArchiverLoad', 'threat_indicator'); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts b/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts index 7e41ed68f8de..957bccd78d1d 100644 --- a/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts +++ b/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts @@ -17,7 +17,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const supertest = getService('supertest'); const testSubjects = getService('testSubjects'); - describe('Data View Management', function () { + // Error: self-signed certificate in certificate chain + describe.skip('Data View Management', function () { let dataViewId = ''; before(async () => { From a588e64a7263613c399a3072bf2656fbb118c5d9 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 28 Aug 2023 10:04:44 +0200 Subject: [PATCH 136/182] fix --- .../cypress/e2e/explore/dashboards/enable_risk_score.cy.ts | 4 ++-- .../test_suites/observability/cypress/e2e/navigation.cy.ts | 3 ++- .../test_suites/security/ftr/cases/attachment_framework.ts | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts index b0ce08a310e6..47a9218981ba 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/enable_risk_score.cy.ts @@ -55,7 +55,7 @@ describe('Enable risk scores', { tags: ['@ess', '@serverless'] }, () => { cy.get(ENABLE_HOST_RISK_SCORE_BUTTON).should('exist'); }); - it('should install host risk score successfully', { tags: ['@brokenInServerless']} () => { + it('should install host risk score successfully', { tags: ['@brokenInServerless'] }, () => { interceptInstallRiskScoreModule(); clickEnableRiskScore(RiskScoreEntity.host); waitForInstallRiskScoreModule(); @@ -89,7 +89,7 @@ describe('Enable risk scores', { tags: ['@ess', '@serverless'] }, () => { cy.get(ENABLE_USER_RISK_SCORE_BUTTON).should('exist'); }); - it('should install user risk score successfully', { tags: ['@brokenInServerless']} () => { + it('should install user risk score successfully', { tags: ['@brokenInServerless'] }, () => { interceptInstallRiskScoreModule(); clickEnableRiskScore(RiskScoreEntity.user); waitForInstallRiskScoreModule(); diff --git a/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts b/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts index df1bea1e56e8..83afaf065d76 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts @@ -5,7 +5,8 @@ * 2.0. */ -describe('Serverless', () => { +// Error: socket hang up +describe.skip('Serverless', () => { it('Should login', () => { cy.loginAsElasticUser(); }); diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts index a35787cff6aa..9a8715229e2e 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts @@ -46,7 +46,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { ); }); - it('adds lens visualization to a new case', async () => { + it.skip('adds lens visualization to a new case', async () => { const caseTitle = 'case created in security solution from my dashboard with lens visualization'; From f699fab5446635988c633602f4513e110a48c744 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 28 Aug 2023 11:03:31 +0200 Subject: [PATCH 137/182] skip failing tests --- .../test_suites/common/encrypted_saved_objects.ts | 3 ++- .../test_suites/observability/threshold_rule/avg_pct_fired.ts | 3 ++- .../observability/discover_log_explorer/dataset_selector.ts | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts index be5dc924c839..86de9673d443 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts @@ -11,7 +11,8 @@ export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); - describe('encrypted saved objects', function () { + // Error: self-signed certificate in certificate chain + describe.skip('encrypted saved objects', function () { describe('route access', () => { describe('disabled', () => { it('rotate key', async () => { diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts index 4830f4915b37..8f32ae0d4178 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts @@ -20,7 +20,8 @@ export default function ({ getService }: FtrProviderContext) { const dataViewApi = getService('dataViewApi'); const logger = getService('log'); - describe('Threshold rule - AVG - PCT - FIRED', () => { + // Error: self-signed certificate in certificate chain + describe.skip('Threshold rule - AVG - PCT - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; diff --git a/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts b/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts index cbb3ea9d95de..9fe3d1e14846 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts @@ -22,7 +22,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const PageObjects = getPageObjects(['common', 'discoverLogExplorer']); - describe('Dataset Selector', () => { + // Error: self-signed certificate in certificate chain + describe.skip('Dataset Selector', () => { before(async () => { await PageObjects.discoverLogExplorer.removeInstalledPackages(); }); From cb2c24cf6888f37fe45e9f7c0def02a58ab72c15 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 28 Aug 2023 12:27:01 +0200 Subject: [PATCH 138/182] skip --- .../cypress/e2e/explore/hosts/host_risk_tab.cy.ts | 3 ++- .../api_integration/test_suites/common/security/anonymous.ts | 3 ++- .../observability/threshold_rule/avg_pct_no_data.ts | 3 ++- .../test_suites/observability/cases/attachment_framework.ts | 3 ++- .../test_suites/security/ftr/cases/attachment_framework.ts | 5 +++-- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/host_risk_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/host_risk_tab.cy.ts index ce59d0de0b9a..6eb6699db551 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/host_risk_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/host_risk_tab.cy.ts @@ -53,7 +53,8 @@ describe('risk tab', { tags: ['@ess', '@brokenInServerless'] }, () => { removeCriticalFilterAndCloseRiskTableFilter(); }); - it('should be able to change items count per page', () => { + // Flaky + it.skip('should be able to change items count per page', () => { selectFiveItemsPerPageOption(); cy.get(HOST_BY_RISK_TABLE_HOSTNAME_CELL).should('have.length', 5); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts index 0b08aa18ce12..407570106787 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts @@ -11,7 +11,8 @@ export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); - describe('security/anonymous', function () { + // Error: self-signed certificate in certificate chain + describe.skip('security/anonymous', function () { describe('route access', () => { describe('disabled', () => { it('get access capabilities', async () => { diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts index 4ff0393e273a..f7ee7e0c5721 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts @@ -17,7 +17,8 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - describe('Threshold rule - AVG - PCT - NoData', () => { + // Error: self-signed certificate in certificate chain + describe.skip('Threshold rule - AVG - PCT - NoData', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id-no-data'; diff --git a/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts index e8be4ff1cf4d..68bda92a8435 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts @@ -19,7 +19,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const cases = getService('cases'); const find = getService('find'); - describe('persistable attachment', () => { + // Error: self-signed certificate in certificate chain + describe.skip('persistable attachment', () => { describe('lens visualization', () => { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts index 9a8715229e2e..d9e50dfed826 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts @@ -18,7 +18,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const cases = getService('cases'); const find = getService('find'); - describe('persistable attachment', () => { + // Failing + describe.skip('persistable attachment', () => { describe('lens visualization', () => { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); @@ -46,7 +47,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { ); }); - it.skip('adds lens visualization to a new case', async () => { + it('adds lens visualization to a new case', async () => { const caseTitle = 'case created in security solution from my dashboard with lens visualization'; From 0d387979667bb7ba10c12b5036e53a8807a77848 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 28 Aug 2023 13:16:49 +0200 Subject: [PATCH 139/182] skip --- .../observability_onboarding/e2e/cypress/e2e/home.cy.ts | 3 ++- .../api_integration/test_suites/common/security/api_keys.ts | 3 ++- .../functional/test_suites/security/ftr/cases/list_view.ts | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts b/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts index 71aa6a0eb09e..6ee30f3eee44 100644 --- a/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts +++ b/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts @@ -5,7 +5,8 @@ * 2.0. */ -describe('[Observability onboarding] Landing page', () => { +// CypressError: `cy.visit()` failed trying to load: +describe.skip('[Observability onboarding] Landing page', () => { beforeEach(() => { cy.loginAsElastic(); }); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts index 256a0fcfe32a..f2f491c95810 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts @@ -13,7 +13,8 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); let roleMapping: { id: string; name: string; api_key: string; encoded: string }; - describe('security/api_keys', function () { + // Error: self-signed certificate in certificate chain + describe.skip('security/api_keys', function () { describe('route access', () => { describe('internal', () => { before(async () => { diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts index 6854b3df6106..e9eca6430448 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts @@ -34,7 +34,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { }); }); - describe('listing', () => { + // Error: self-signed certificate in certificate chain + describe.skip('listing', () => { createNCasesBeforeDeleteAllAfter(2, getPageObject, getService); it('lists cases correctly', async () => { From 6e4b6bd05ef324e6ebb103631e646f7ad76577b9 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 28 Aug 2023 14:24:17 +0200 Subject: [PATCH 140/182] fix --- .buildkite/pipelines/pull_request/defend_workflows.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipelines/pull_request/defend_workflows.yml b/.buildkite/pipelines/pull_request/defend_workflows.yml index 953cc3ab971f..3a50e3ece206 100644 --- a/.buildkite/pipelines/pull_request/defend_workflows.yml +++ b/.buildkite/pipelines/pull_request/defend_workflows.yml @@ -4,7 +4,7 @@ steps: agents: queue: n2-4-spot depends_on: build - timeout_in_minutes: 120 + timeout_in_minutes: 60 parallelism: 2 retry: automatic: @@ -18,8 +18,8 @@ steps: agents: queue: n2-4-virt depends_on: build - timeout_in_minutes: 120 - parallelism: 5 + timeout_in_minutes: 60 + parallelism: 6 retry: automatic: - exit_status: '*' From af4bf1703facb7e9dfd2e06e9450316598b02504 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 28 Aug 2023 15:56:06 +0100 Subject: [PATCH 141/182] chore(NA): include cert authorities on base config --- x-pack/test_serverless/shared/config.base.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index eb1390544203..f199500ea204 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -18,6 +18,7 @@ import { } from '@kbn/test'; import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; +import { createKibanaSupertestProvider } from '../../../test/server_integration/services'; export default async () => { const servers = { @@ -138,6 +139,9 @@ export default async () => { services: { ...commonFunctionalServices, + supertest: createKibanaSupertestProvider({ + certificateAuthorities: [Fs.readFileSync(CA_CERT_PATH)], + }), }, // overriding default timeouts from packages/kbn-test/src/functional_test_runner/lib/config/schema.ts From 12fa58d774b9a5c892a31ef1533a7b30c419b484 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 28 Aug 2023 16:31:27 +0100 Subject: [PATCH 142/182] fix(NA): base config supertest service --- x-pack/test_serverless/shared/config.base.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index f199500ea204..c2f1f2dcb1a1 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -8,6 +8,7 @@ import { resolve } from 'path'; import { format as formatUrl } from 'url'; import Fs from 'fs'; +import supertest from 'supertest'; import { REPO_ROOT } from '@kbn/repo-info'; import { @@ -18,7 +19,6 @@ import { } from '@kbn/test'; import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; -import { createKibanaSupertestProvider } from '../../../test/server_integration/services'; export default async () => { const servers = { @@ -139,9 +139,9 @@ export default async () => { services: { ...commonFunctionalServices, - supertest: createKibanaSupertestProvider({ - certificateAuthorities: [Fs.readFileSync(CA_CERT_PATH)], - }), + // TODO: this can be abstracted into @kbn/ftr-common-functional-services + // We can use the implementation at test/server_integration/services in the function createKibanaSupertestProvider + supertest: supertest.agent(formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] })), }, // overriding default timeouts from packages/kbn-test/src/functional_test_runner/lib/config/schema.ts From b1143577baae830370f208ee7daa498a861a586b Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 28 Aug 2023 15:36:43 +0000 Subject: [PATCH 143/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- x-pack/test_serverless/shared/config.base.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index c2f1f2dcb1a1..f09403c75935 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -141,7 +141,9 @@ export default async () => { ...commonFunctionalServices, // TODO: this can be abstracted into @kbn/ftr-common-functional-services // We can use the implementation at test/server_integration/services in the function createKibanaSupertestProvider - supertest: supertest.agent(formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] })), + supertest: supertest.agent( + formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] }) + ), }, // overriding default timeouts from packages/kbn-test/src/functional_test_runner/lib/config/schema.ts From 3ffdefae2c6d2ec59f40b5d578774a6f2747a797 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 28 Aug 2023 19:19:07 +0100 Subject: [PATCH 144/182] fix(NA): introduce function for supertest service --- x-pack/test_serverless/shared/config.base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index c2f1f2dcb1a1..3955709801fc 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -141,7 +141,7 @@ export default async () => { ...commonFunctionalServices, // TODO: this can be abstracted into @kbn/ftr-common-functional-services // We can use the implementation at test/server_integration/services in the function createKibanaSupertestProvider - supertest: supertest.agent(formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] })), + supertest: ({ getService }) => supertest.agent(formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] })), }, // overriding default timeouts from packages/kbn-test/src/functional_test_runner/lib/config/schema.ts From fd34dc294f5115da7f238eb4bae1127e4c176a23 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:26:41 +0000 Subject: [PATCH 145/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- x-pack/test_serverless/shared/config.base.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index 280d860b1c92..bd5d81971054 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -141,7 +141,8 @@ export default async () => { ...commonFunctionalServices, // TODO: this can be abstracted into @kbn/ftr-common-functional-services // We can use the implementation at test/server_integration/services in the function createKibanaSupertestProvider - supertest: ({getService}) => supertest.agent(formatUrl(servers.kibana, {ca: [Fs.readFileSync(CA_CERT_PATH)]})), + supertest: ({ getService }) => + supertest.agent(formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] })), }, // overriding default timeouts from packages/kbn-test/src/functional_test_runner/lib/config/schema.ts From bd81652b5accb659d1cb6d204e32b2a7af5ed121 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 28 Aug 2023 23:29:03 +0100 Subject: [PATCH 146/182] chore(NA): introduced supertest for serverless api_integration --- .../common/security/authentication.ts | 9 ++--- x-pack/test_serverless/shared/config.base.ts | 7 ++-- .../test_serverless/shared/services/index.ts | 6 +++- .../shared/services/supertest.ts | 33 +++++++++++++++++++ 4 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 x-pack/test_serverless/shared/services/supertest.ts diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts index 590adbe267b4..6bd01780587f 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/authentication.ts @@ -134,16 +134,17 @@ export default function ({ getService }: FtrProviderContext) { // expect success because we're using the internal header expect(body).toEqual({ authentication_provider: { name: '__http__', type: 'http' }, - authentication_realm: { name: 'reserved', type: 'reserved' }, + authentication_realm: { name: 'file1', type: 'file' }, authentication_type: 'realm', elastic_cloud_user: false, email: null, enabled: true, full_name: null, - lookup_realm: { name: 'reserved', type: 'reserved' }, - metadata: { _reserved: true }, + lookup_realm: { name: 'file1', type: 'file' }, + metadata: {}, + operator: true, roles: ['superuser'], - username: 'elastic', + username: 'elastic_serverless', }); expect(status).toBe(200); }); diff --git a/x-pack/test_serverless/shared/config.base.ts b/x-pack/test_serverless/shared/config.base.ts index bd5d81971054..83504821b70f 100644 --- a/x-pack/test_serverless/shared/config.base.ts +++ b/x-pack/test_serverless/shared/config.base.ts @@ -8,7 +8,6 @@ import { resolve } from 'path'; import { format as formatUrl } from 'url'; import Fs from 'fs'; -import supertest from 'supertest'; import { REPO_ROOT } from '@kbn/repo-info'; import { @@ -19,6 +18,7 @@ import { } from '@kbn/test'; import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils'; import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; +import { services } from './services'; export default async () => { const servers = { @@ -139,10 +139,7 @@ export default async () => { services: { ...commonFunctionalServices, - // TODO: this can be abstracted into @kbn/ftr-common-functional-services - // We can use the implementation at test/server_integration/services in the function createKibanaSupertestProvider - supertest: ({ getService }) => - supertest.agent(formatUrl(servers.kibana, { ca: [Fs.readFileSync(CA_CERT_PATH)] })), + ...services, }, // overriding default timeouts from packages/kbn-test/src/functional_test_runner/lib/config/schema.ts diff --git a/x-pack/test_serverless/shared/services/index.ts b/x-pack/test_serverless/shared/services/index.ts index d6e9fd713d90..3bca21f9694b 100644 --- a/x-pack/test_serverless/shared/services/index.ts +++ b/x-pack/test_serverless/shared/services/index.ts @@ -5,4 +5,8 @@ * 2.0. */ -export const services = {}; +import {SupertestProvider, SupertestWithoutAuthProvider} from './supertest'; +export const services = { + supertest: SupertestProvider, + supertestWithoutAuth: SupertestWithoutAuthProvider +}; diff --git a/x-pack/test_serverless/shared/services/supertest.ts b/x-pack/test_serverless/shared/services/supertest.ts new file mode 100644 index 000000000000..ba8111153b23 --- /dev/null +++ b/x-pack/test_serverless/shared/services/supertest.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { format as formatUrl } from 'url'; +import { FtrProviderContext } from '../../functional/ftr_provider_context'; + +import supertest from 'supertest'; + +export function SupertestProvider({ getService }: FtrProviderContext) { + const config = getService('config'); + const kbnUrl = formatUrl(config.get('servers.kibana')); + const cAuthorities = config.get('servers.kibana').certificateAuthorities + + return supertest.agent(kbnUrl, { ca: cAuthorities }); +} + +export function SupertestWithoutAuthProvider({ getService }: FtrProviderContext) { + const config = getService('config'); + const kbnUrl = formatUrl({ + ...config.get('servers.kibana'), + auth: false, + }); + const cAuthorities = config.get('servers.kibana').certificateAuthorities + + return supertest.agent(kbnUrl, + { ca: cAuthorities } + ); +} From 59082e163852f8ac6bc3a3381d32db75b78c3be4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 28 Aug 2023 22:33:30 +0000 Subject: [PATCH 147/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../test_serverless/shared/services/index.ts | 4 ++-- .../shared/services/supertest.ts | 22 ++++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/x-pack/test_serverless/shared/services/index.ts b/x-pack/test_serverless/shared/services/index.ts index 3bca21f9694b..02a03229b838 100644 --- a/x-pack/test_serverless/shared/services/index.ts +++ b/x-pack/test_serverless/shared/services/index.ts @@ -5,8 +5,8 @@ * 2.0. */ -import {SupertestProvider, SupertestWithoutAuthProvider} from './supertest'; +import { SupertestProvider, SupertestWithoutAuthProvider } from './supertest'; export const services = { supertest: SupertestProvider, - supertestWithoutAuth: SupertestWithoutAuthProvider + supertestWithoutAuth: SupertestWithoutAuthProvider, }; diff --git a/x-pack/test_serverless/shared/services/supertest.ts b/x-pack/test_serverless/shared/services/supertest.ts index ba8111153b23..3855bbdf7137 100644 --- a/x-pack/test_serverless/shared/services/supertest.ts +++ b/x-pack/test_serverless/shared/services/supertest.ts @@ -1,22 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ import { format as formatUrl } from 'url'; -import { FtrProviderContext } from '../../functional/ftr_provider_context'; - import supertest from 'supertest'; +import { FtrProviderContext } from '../../functional/ftr_provider_context'; export function SupertestProvider({ getService }: FtrProviderContext) { - const config = getService('config'); - const kbnUrl = formatUrl(config.get('servers.kibana')); - const cAuthorities = config.get('servers.kibana').certificateAuthorities + const config = getService('config'); + const kbnUrl = formatUrl(config.get('servers.kibana')); + const cAuthorities = config.get('servers.kibana').certificateAuthorities; - return supertest.agent(kbnUrl, { ca: cAuthorities }); + return supertest.agent(kbnUrl, { ca: cAuthorities }); } export function SupertestWithoutAuthProvider({ getService }: FtrProviderContext) { @@ -25,9 +23,7 @@ export function SupertestWithoutAuthProvider({ getService }: FtrProviderContext) ...config.get('servers.kibana'), auth: false, }); - const cAuthorities = config.get('servers.kibana').certificateAuthorities + const cAuthorities = config.get('servers.kibana').certificateAuthorities; - return supertest.agent(kbnUrl, - { ca: cAuthorities } - ); + return supertest.agent(kbnUrl, { ca: cAuthorities }); } From c3211db44e09c8098f11f432a17be4bcea2df5dd Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 28 Aug 2023 23:50:12 +0100 Subject: [PATCH 148/182] chore(NA): skip alerts cypress serverless test --- .../cypress/e2e/detection_alerts/ransomware_prevention.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts index ea8074389868..2bde88759ad6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts @@ -34,7 +34,7 @@ describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () = waitForAlertsToPopulate(); }); - describe('Alerts table', () => { + describe('Alerts table', { tags: ['@brokenInServerless'] }, () => { it('shows Ransomware Alerts', () => { cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Prevention Alert'); }); From 1467ca085f128f0319d98d8cad40b9bf57466e83 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 20:50:53 -0600 Subject: [PATCH 149/182] Split jwt esargs for ess to fix docker ssl --- packages/kbn-es/src/utils/docker.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index c5fb46ca7dc5..381d3c13769d 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -153,7 +153,9 @@ const DEFAULT_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.transport.ssl.verification_mode', 'certificate'], ['xpack.security.operator_privileges.enabled', 'true'], +]; +const SERVERLESS_SSL_ESARGS: Array<[string, string]> = [ ['xpack.security.authc.realms.jwt.jwt1.client_authentication.type', 'shared_secret'], ['xpack.security.authc.realms.jwt.jwt1.order', '-98'], @@ -535,7 +537,13 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO ...node, image, params: node.params.concat( - resolveEsArgs(DEFAULT_SERVERLESS_ESARGS.concat(node.esArgs ?? []), options), + resolveEsArgs( + DEFAULT_SERVERLESS_ESARGS.concat( + node.esArgs ?? [], + options.ssl ? SERVERLESS_SSL_ESARGS : [] + ), + options + ), i === 0 ? resolvePort(options) : [], volumeCmd ), From a97f64ca9b03e3805d423577106fda73c6878b44 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 04:05:45 +0100 Subject: [PATCH 150/182] fix(NA): test_serverless observability config --- .../observability/cypress/cypress.config.ts | 14 +++++++------- .../observability/cypress/e2e/navigation.cy.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/cypress/cypress.config.ts b/x-pack/test_serverless/functional/test_suites/observability/cypress/cypress.config.ts index 52eaff1a6779..b570ec174db3 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cypress/cypress.config.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cypress/cypress.config.ts @@ -6,14 +6,14 @@ */ import { defineCypressConfig } from '@kbn/cypress-config'; -import { kbnTestConfig } from '@kbn/test'; +import { kbnTestConfig, kibanaTestSuperuserServerless } from '@kbn/test'; import Url from 'url'; const kibanaUrlWithoutAuth = Url.format({ - protocol: kbnTestConfig.getUrlParts().protocol, - hostname: kbnTestConfig.getUrlParts().hostname, - port: kbnTestConfig.getUrlParts().port, + protocol: 'https', + hostname: kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).hostname, + port: kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).port, }); export default defineCypressConfig({ @@ -35,13 +35,13 @@ export default defineCypressConfig({ runMode: 1, }, e2e: { - baseUrl: 'http://localhost:5620', + baseUrl: 'https://localhost:5620', supportFile: './support/e2e.ts', specPattern: './e2e/**/*.cy.ts', }, env: { - username: kbnTestConfig.getUrlParts().username, - password: kbnTestConfig.getUrlParts().password, + username: kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).username, + password: kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).password, kibanaUrlWithoutAuth, }, }); diff --git a/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts b/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts index 51a381108408..b6a35b09acef 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts @@ -6,7 +6,7 @@ */ // Error: socket hang up -describe.skip('Serverless', () => { +describe('Serverless', () => { beforeEach(() => { cy.loginAsElasticUser(); }); From 066893da07dcc716ec62bee24e2b4134caa9dc32 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 04:10:18 +0100 Subject: [PATCH 151/182] fix(NA): add on more brokenInServerless tag --- .../cypress/e2e/detection_alerts/ransomware_prevention.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts index 2bde88759ad6..9627dc70f018 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts @@ -40,7 +40,7 @@ describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () = }); }); - describe('Trend Chart', () => { + describe('Trend Chart', { tags: ['@brokenInServerless'] }, () => { beforeEach(() => { selectAlertsHistogram(); }); From 74ab8850f72628e196a55f476a0851877c828fd6 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 04:26:53 +0100 Subject: [PATCH 152/182] fix(NA): a couple more cypress tests --- .../api_integration/test_suites/common/alerting/rules.ts | 2 +- .../test_suites/observability/cases/helpers/api.ts | 2 +- .../api_integration/test_suites/security/cases/helpers/api.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts index bfce78384f60..34ece9a12921 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts @@ -35,7 +35,7 @@ export default function ({ getService }: FtrProviderContext) { const esClient = getService('es'); const esDeleteAllIndices = getService('esDeleteAllIndices'); - describe('Alerting rules', () => { + describe.skip('Alerting rules', () => { const RULE_TYPE_ID = '.es-query'; const ALERT_ACTION_INDEX = 'alert-action-es-query'; let actionId: string; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/cases/helpers/api.ts b/x-pack/test_serverless/api_integration/test_suites/observability/cases/helpers/api.ts index 5f196ef3e337..33cbdf07bf60 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/cases/helpers/api.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/cases/helpers/api.ts @@ -112,7 +112,7 @@ export const deleteMappings = async (es: Client): Promise => { }); }; -export const defaultUser = { email: null, full_name: null, username: 'elastic' }; +export const defaultUser = { email: null, full_name: null, username: 'elastic_serverless' }; /** * A null filled user will occur when the security plugin is disabled */ diff --git a/x-pack/test_serverless/api_integration/test_suites/security/cases/helpers/api.ts b/x-pack/test_serverless/api_integration/test_suites/security/cases/helpers/api.ts index afba9c46c67c..0d1a889adead 100644 --- a/x-pack/test_serverless/api_integration/test_suites/security/cases/helpers/api.ts +++ b/x-pack/test_serverless/api_integration/test_suites/security/cases/helpers/api.ts @@ -112,7 +112,7 @@ export const deleteMappings = async (es: Client): Promise => { }); }; -export const defaultUser = { email: null, full_name: null, username: 'elastic' }; +export const defaultUser = { email: null, full_name: null, username: 'elastic_serverless' }; /** * A null filled user will occur when the security plugin is disabled */ From 60c77427b113dbd18bacec726cf31f7d3bc99150 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 05:02:31 +0100 Subject: [PATCH 153/182] chore(NA): skip one more test --- .../cypress/e2e/detection_alerts/ransomware_detection.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts index 6eb98a8c11d7..dc4006558c18 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts @@ -48,7 +48,7 @@ describe( }); }); - describe('Ransomware in Timelines', () => { + describe('Ransomware in Timelines', { tags: ['@brokenInServerless'] }, () => { before(() => { login(); visit(TIMELINES_URL); From 772312e8802b05c18c1067ce80bd1745db4efadf Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 05:04:19 +0100 Subject: [PATCH 154/182] chore(NA): skip one more test --- .../cypress/e2e/detection_alerts/ransomware_detection.cy.ts | 2 +- .../cypress/e2e/detection_alerts/ransomware_prevention.cy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts index dc4006558c18..6eb98a8c11d7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_detection.cy.ts @@ -48,7 +48,7 @@ describe( }); }); - describe('Ransomware in Timelines', { tags: ['@brokenInServerless'] }, () => { + describe('Ransomware in Timelines', () => { before(() => { login(); visit(TIMELINES_URL); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts index 9627dc70f018..76dff3c72441 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts @@ -51,7 +51,7 @@ describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () = }); }); - describe('Ransomware in Timelines', () => { + describe('Ransomware in Timelines', { tags: ['@brokenInServerless'] }, () => { beforeEach(() => { login(); visit(TIMELINES_URL); From 7661fba36c0c6b801dcca113b2df99eb10ccefb4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 29 Aug 2023 04:08:46 +0000 Subject: [PATCH 155/182] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../cypress/e2e/detection_alerts/ransomware_prevention.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts index 76dff3c72441..78bb2686b102 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts @@ -51,7 +51,7 @@ describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () = }); }); - describe('Ransomware in Timelines', { tags: ['@brokenInServerless'] }, () => { + describe('Ransomware in Timelines', { tags: ['@brokenInServerless'] }, () => { beforeEach(() => { login(); visit(TIMELINES_URL); From 9041e77b1817243e91ecc14f7c2df5f48117be0f Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:20:12 -0600 Subject: [PATCH 156/182] Remove cert skip - fixed --- .../test_suites/common/encrypted_saved_objects.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts index 86de9673d443..be5dc924c839 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/encrypted_saved_objects.ts @@ -11,8 +11,7 @@ export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); - // Error: self-signed certificate in certificate chain - describe.skip('encrypted saved objects', function () { + describe('encrypted saved objects', function () { describe('route access', () => { describe('disabled', () => { it('rotate key', async () => { From 337da797497516c89f9191e5c043474dd4bab650 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:21:27 -0600 Subject: [PATCH 157/182] Remove cert skip - fixed --- .../api_integration/test_suites/common/security/anonymous.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts index 407570106787..0b08aa18ce12 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/anonymous.ts @@ -11,8 +11,7 @@ export default function ({ getService }: FtrProviderContext) { const svlCommonApi = getService('svlCommonApi'); const supertest = getService('supertest'); - // Error: self-signed certificate in certificate chain - describe.skip('security/anonymous', function () { + describe('security/anonymous', function () { describe('route access', () => { describe('disabled', () => { it('get access capabilities', async () => { From 76e8559d75f2770c406052666dc377aacc15172d Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:23:05 -0600 Subject: [PATCH 158/182] Remove cert skip - fixed --- .../api_integration/test_suites/common/security/api_keys.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts index f2f491c95810..256a0fcfe32a 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/security/api_keys.ts @@ -13,8 +13,7 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); let roleMapping: { id: string; name: string; api_key: string; encoded: string }; - // Error: self-signed certificate in certificate chain - describe.skip('security/api_keys', function () { + describe('security/api_keys', function () { describe('route access', () => { describe('internal', () => { before(async () => { From 87105784b4f55de128ed520f31b6a47e489f7add Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:36:20 -0600 Subject: [PATCH 159/182] Change skip reason --- .../test_suites/observability/threshold_rule/avg_pct_fired.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts index 8f32ae0d4178..8d872970b640 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts @@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { const dataViewApi = getService('dataViewApi'); const logger = getService('log'); - // Error: self-signed certificate in certificate chain + // Blocked API: index_not_found_exception: no such index [.alerts-observability.threshold.alerts-default] describe.skip('Threshold rule - AVG - PCT - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; From e9d16a0a8872b92416f380d603afc5690f364829 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:41:06 -0600 Subject: [PATCH 160/182] Change skip reason --- .../test_suites/observability/threshold_rule/avg_pct_no_data.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts index f7ee7e0c5721..a3d8f023804b 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts @@ -17,7 +17,7 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - // Error: self-signed certificate in certificate chain + // Blocked API: index_not_found_exception: no such index [.alerts-observability.threshold.alerts-default] describe.skip('Threshold rule - AVG - PCT - NoData', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; From bb850b787f8c113a8953eb0d99b7ca6610047082 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:44:17 -0600 Subject: [PATCH 161/182] Remove cert skip - fixed --- .../functional/test_suites/common/data_view_mgmt.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts b/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts index 957bccd78d1d..7e41ed68f8de 100644 --- a/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts +++ b/x-pack/test_serverless/functional/test_suites/common/data_view_mgmt.ts @@ -17,8 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const supertest = getService('supertest'); const testSubjects = getService('testSubjects'); - // Error: self-signed certificate in certificate chain - describe.skip('Data View Management', function () { + describe('Data View Management', function () { let dataViewId = ''; before(async () => { From 1e3040790b43a703e200c52e069ef2384a3c741d Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 22:47:39 -0600 Subject: [PATCH 162/182] Remove cert skip - fixed --- .../test_suites/observability/cases/attachment_framework.ts | 3 +-- .../test_suites/observability/cypress/e2e/navigation.cy.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts index 68bda92a8435..e8be4ff1cf4d 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cases/attachment_framework.ts @@ -19,8 +19,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const cases = getService('cases'); const find = getService('find'); - // Error: self-signed certificate in certificate chain - describe.skip('persistable attachment', () => { + describe('persistable attachment', () => { describe('lens visualization', () => { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); diff --git a/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts b/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts index b6a35b09acef..84abae3258cf 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/cypress/e2e/navigation.cy.ts @@ -5,7 +5,6 @@ * 2.0. */ -// Error: socket hang up describe('Serverless', () => { beforeEach(() => { cy.loginAsElasticUser(); From 9289a75e6540d6e34fc53d6d900be596e7fd938d Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 23:08:15 -0600 Subject: [PATCH 163/182] Remove cert skip - fixed. Skip failure --- .../observability/discover_log_explorer/dataset_selector.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts b/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts index 9fe3d1e14846..5a1f9946b898 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts @@ -22,8 +22,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const PageObjects = getPageObjects(['common', 'discoverLogExplorer']); - // Error: self-signed certificate in certificate chain - describe.skip('Dataset Selector', () => { + describe('Dataset Selector', () => { before(async () => { await PageObjects.discoverLogExplorer.removeInstalledPackages(); }); @@ -84,7 +83,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } }); - it('should display an empty prompt for no integrations', async () => { + // Skip: failing assertion + it.skip('should display an empty prompt for no integrations', async () => { const { integrations } = await PageObjects.discoverLogExplorer.getIntegrations(); expect(integrations.length).to.be(0); From 18ed3ea9448544aa4673d6ec051e77c1afac4dc2 Mon Sep 17 00:00:00 2001 From: Brad White Date: Mon, 28 Aug 2023 23:23:13 -0600 Subject: [PATCH 164/182] Add skip --- .../functional/test_suites/security/ftr/cases/list_view.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts index e9eca6430448..2d887da4fa41 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts @@ -16,7 +16,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const cases = getService('cases'); const svlSecNavigation = getService('svlSecNavigation'); - describe('cases list', () => { + // Failing + describe.skip('cases list', () => { before(async () => { await svlSecNavigation.navigateToLandingPage(); @@ -35,7 +36,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { }); // Error: self-signed certificate in certificate chain - describe.skip('listing', () => { + describe('listing', () => { createNCasesBeforeDeleteAllAfter(2, getPageObject, getService); it('lists cases correctly', async () => { From 0e6314e0e808b90354b705c2b46dc1cfcfdf5119 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 29 Aug 2023 08:50:19 +0200 Subject: [PATCH 165/182] fix --- packages/kbn-es/src/utils/docker.test.ts | 14 +--- .../ransomware_prevention.cy.ts | 80 ++++++++++--------- 2 files changed, 43 insertions(+), 51 deletions(-) diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index 510619976c41..c42ac1af577f 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -341,7 +341,7 @@ describe('resolveEsArgs()', () => { ssl: true, }); - expect(esArgs).toHaveLength(32); + expect(esArgs).toHaveLength(20); expect(esArgs).not.toEqual(expect.arrayContaining(['xpack.security.enabled=false'])); expect(esArgs).toMatchInlineSnapshot(` Array [ @@ -365,18 +365,6 @@ describe('resolveEsArgs()', () => { "xpack.security.transport.ssl.verification_mode=certificate", "--env", "xpack.security.operator_privileges.enabled=true", - "--env", - "xpack.security.authc.realms.jwt.jwt1.client_authentication.type=shared_secret", - "--env", - "xpack.security.authc.realms.jwt.jwt1.order=-98", - "--env", - "xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/", - "--env", - "xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch", - "--env", - "xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path=/usr/share/elasticsearch/config/secrets/jwks.json", - "--env", - "xpack.security.authc.realms.jwt.jwt1.claims.principal=sub", ] `); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts index 78bb2686b102..6ce7f88b8d83 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_alerts/ransomware_prevention.cy.ts @@ -15,57 +15,61 @@ import { selectAlertsHistogram } from '../../tasks/alerts'; import { createTimeline } from '../../tasks/timelines'; import { cleanKibana } from '../../tasks/common'; -describe('Ransomware Prevention Alerts', { tags: ['@ess', '@serverless'] }, () => { - before(() => { - cleanKibana(); - cy.task('esArchiverLoad', { - archiveName: 'ransomware_prevention', +describe( + 'Ransomware Prevention Alerts', + { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + () => { + before(() => { + cleanKibana(); + cy.task('esArchiverLoad', { + archiveName: 'ransomware_prevention', + }); }); - }); - - after(() => { - cy.task('esArchiverUnload', 'ransomware_prevention'); - }); - describe('Ransomware display in Alerts Section', () => { - beforeEach(() => { - login(); - visit(ALERTS_URL); - waitForAlertsToPopulate(); + after(() => { + cy.task('esArchiverUnload', 'ransomware_prevention'); }); - describe('Alerts table', { tags: ['@brokenInServerless'] }, () => { - it('shows Ransomware Alerts', () => { - cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Prevention Alert'); + describe('Ransomware display in Alerts Section', () => { + beforeEach(() => { + login(); + visit(ALERTS_URL); + waitForAlertsToPopulate(); }); - }); - describe('Trend Chart', { tags: ['@brokenInServerless'] }, () => { - beforeEach(() => { - selectAlertsHistogram(); + describe('Alerts table', () => { + it('shows Ransomware Alerts', () => { + cy.get(ALERT_RULE_NAME).should('have.text', 'Ransomware Prevention Alert'); + }); }); - it('shows Ransomware Prevention Alert in the trend chart', () => { - cy.get(ALERTS_HISTOGRAM_SERIES).should('have.text', 'Ransomware Prevention Alert'); + describe('Trend Chart', () => { + beforeEach(() => { + selectAlertsHistogram(); + }); + + it('shows Ransomware Prevention Alert in the trend chart', () => { + cy.get(ALERTS_HISTOGRAM_SERIES).should('have.text', 'Ransomware Prevention Alert'); + }); }); }); - }); - describe('Ransomware in Timelines', { tags: ['@brokenInServerless'] }, () => { - beforeEach(() => { - login(); - visit(TIMELINES_URL); + describe('Ransomware in Timelines', () => { + beforeEach(() => { + login(); + visit(TIMELINES_URL); - createTimeline(); - }); + createTimeline(); + }); - it('Renders ransomware entries in timelines table', () => { - cy.get(TIMELINE_QUERY).type('event.code: "ransomware"{enter}'); + it('Renders ransomware entries in timelines table', () => { + cy.get(TIMELINE_QUERY).type('event.code: "ransomware"{enter}'); - // Wait for grid to load, it should have an analyzer icon - cy.get(TIMELINE_VIEW_IN_ANALYZER).should('exist'); + // Wait for grid to load, it should have an analyzer icon + cy.get(TIMELINE_VIEW_IN_ANALYZER).should('exist'); - cy.get(MESSAGE).should('have.text', 'Ransomware Prevention Alert'); + cy.get(MESSAGE).should('have.text', 'Ransomware Prevention Alert'); + }); }); - }); -}); + } +); From 5d8ebd6992b98a9cdaf38f42b91328fc77f34d32 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 13:35:53 +0100 Subject: [PATCH 166/182] skipping api_integration tests from obs --- .../observability/threshold_rule/custom_eq_avg_bytes_fired.ts | 2 +- .../observability/threshold_rule/documents_count_fired.ts | 2 +- .../test_suites/observability/threshold_rule/group_by_fired.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts index 7b2aea23f238..b0607471f46a 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts @@ -26,7 +26,7 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - describe('Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { + describe.skip('Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts index bd1fed6a6bd5..dd95ef7e80dc 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts @@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - describe('Threshold rule - DOCUMENTS_COUNT - FIRED', () => { + describe.skip('Threshold rule - DOCUMENTS_COUNT - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts index 244656dd97d9..4a8b8cb65bae 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts @@ -30,7 +30,7 @@ export default function ({ getService }: FtrProviderContext) { let alertId: string; let startedAt: string; - describe('Threshold rule - GROUP_BY - FIRED', () => { + describe.skip('Threshold rule - GROUP_BY - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; From 9f1942319101698b39a34c7c99188bd4ea5c49ee Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 15:19:31 +0100 Subject: [PATCH 167/182] fix(NA): skip failing tests on apm_api_integration/feature_flags.ts --- .../observability/apm_api_integration/feature_flags.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts b/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts index 44e188c0402e..fc951f0ecde4 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts @@ -67,7 +67,7 @@ async function uploadSourcemap(apmApiClient: any) { export default function ({ getService }: APMFtrContextProvider) { const apmApiClient = getService('apmApiClient'); - describe('apm feature flags', () => { + describe.skip('apm feature flags', () => { describe('fleet migrations', () => { it('rejects requests to save apm server schema', async () => { try { From 82924a62450b76da6bebc1ae53537644a78356a6 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 15:30:53 +0100 Subject: [PATCH 168/182] chore(NA): unskip cases/list_view.ts test --- .../functional/test_suites/security/ftr/cases/list_view.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts index 2d887da4fa41..9cead3565d18 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts @@ -17,7 +17,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const svlSecNavigation = getService('svlSecNavigation'); // Failing - describe.skip('cases list', () => { + describe('cases list', () => { before(async () => { await svlSecNavigation.navigateToLandingPage(); @@ -35,7 +35,6 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { }); }); - // Error: self-signed certificate in certificate chain describe('listing', () => { createNCasesBeforeDeleteAllAfter(2, getPageObject, getService); From 1765ad046e88d9f8ff040feb92fa1eb6d04e8cf0 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 15:31:43 +0100 Subject: [PATCH 169/182] chore(NA): remove failing comment --- .../functional/test_suites/security/ftr/cases/list_view.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts index 9cead3565d18..1af492bdc06a 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/list_view.ts @@ -15,8 +15,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const testSubjects = getService('testSubjects'); const cases = getService('cases'); const svlSecNavigation = getService('svlSecNavigation'); - - // Failing + describe('cases list', () => { before(async () => { await svlSecNavigation.navigateToLandingPage(); From ebd3a95bf9b7a14d37ea9663ab3ba9c19d9fc076 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 15:45:02 +0100 Subject: [PATCH 170/182] chore(NA): unskip home.cy.ts --- .../observability_onboarding/e2e/cypress/e2e/home.cy.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts b/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts index 6ee30f3eee44..71aa6a0eb09e 100644 --- a/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts +++ b/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts @@ -5,8 +5,7 @@ * 2.0. */ -// CypressError: `cy.visit()` failed trying to load: -describe.skip('[Observability onboarding] Landing page', () => { +describe('[Observability onboarding] Landing page', () => { beforeEach(() => { cy.loginAsElastic(); }); From 9ed26920818fd2077e27b588e705fddceca13c6f Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 17:18:22 +0100 Subject: [PATCH 171/182] fix(NA): add @brokenInServerless to cell_actions.cy.ts --- .../e2e/investigations/timelines/discover/cell_actions.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/cell_actions.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/cell_actions.cy.ts index 7eb818ef9205..0325117e6c01 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/cell_actions.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/cell_actions.cy.ts @@ -52,7 +52,7 @@ describe( ); }); }); - it('Filter out', () => { + it('Filter out', { tags: ['@brokenInServerless'] }, () => { cy.get(GET_DISCOVER_DATA_GRID_CELL(TIMESTAMP_COLUMN_NAME, 0)).then((sub) => { const selectedTimestamp = sub.text(); cy.get(GET_DISCOVER_DATA_GRID_CELL(TIMESTAMP_COLUMN_NAME, 0)).realHover(); From 93bcc147d2fe1c1b28f2cf0bd441b2873ef49ec6 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 29 Aug 2023 11:16:26 -0600 Subject: [PATCH 172/182] Add security threat hunting issue --- .../test_suites/security/ftr/cases/attachment_framework.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts index 978b0d4a8a07..c213e48348b6 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cases/attachment_framework.ts @@ -19,6 +19,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const find = getService('find'); // Failing + // Issue: https://github.com/elastic/kibana/issues/165135 describe.skip('persistable attachment', () => { describe('lens visualization', () => { before(async () => { From 3629766589d0e226d7a43deeea644d12c7ba323a Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 29 Aug 2023 11:20:55 -0600 Subject: [PATCH 173/182] Add observability issue --- .../observability/discover_log_explorer/dataset_selector.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts b/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts index 5a1f9946b898..124bacb9ac1f 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/discover_log_explorer/dataset_selector.ts @@ -84,6 +84,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); // Skip: failing assertion + // Issue: https://github.com/elastic/kibana/issues/165138 it.skip('should display an empty prompt for no integrations', async () => { const { integrations } = await PageObjects.discoverLogExplorer.getIntegrations(); expect(integrations.length).to.be(0); From 6d37b5b5cca90025a5b5527ac29a671e4f7b5cd1 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 29 Aug 2023 11:26:11 -0600 Subject: [PATCH 174/182] More obs issues --- .../observability/apm_api_integration/feature_flags.ts | 1 + .../test_suites/observability/threshold_rule/avg_pct_fired.ts | 1 + .../test_suites/observability/threshold_rule/avg_pct_no_data.ts | 1 + .../observability/threshold_rule/custom_eq_avg_bytes_fired.ts | 1 + .../observability/threshold_rule/documents_count_fired.ts | 1 + .../test_suites/observability/threshold_rule/group_by_fired.ts | 1 + 6 files changed, 6 insertions(+) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts b/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts index fc951f0ecde4..93c621c72af7 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/feature_flags.ts @@ -67,6 +67,7 @@ async function uploadSourcemap(apmApiClient: any) { export default function ({ getService }: APMFtrContextProvider) { const apmApiClient = getService('apmApiClient'); + // Issue: https://github.com/elastic/kibana/issues/165138 describe.skip('apm feature flags', () => { describe('fleet migrations', () => { it('rejects requests to save apm server schema', async () => { diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts index 8d872970b640..48256d153b98 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts @@ -21,6 +21,7 @@ export default function ({ getService }: FtrProviderContext) { const logger = getService('log'); // Blocked API: index_not_found_exception: no such index [.alerts-observability.threshold.alerts-default] + // Issue: https://github.com/elastic/kibana/issues/165138 describe.skip('Threshold rule - AVG - PCT - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts index a3d8f023804b..3d97a7a09433 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts @@ -18,6 +18,7 @@ export default function ({ getService }: FtrProviderContext) { const dataViewApi = getService('dataViewApi'); // Blocked API: index_not_found_exception: no such index [.alerts-observability.threshold.alerts-default] + // Issue: https://github.com/elastic/kibana/issues/165138 describe.skip('Threshold rule - AVG - PCT - NoData', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts index b0607471f46a..276ac212dac4 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts @@ -26,6 +26,7 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); + // Issue: https://github.com/elastic/kibana/issues/165138 describe.skip('Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts index dd95ef7e80dc..0a771d336379 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts @@ -20,6 +20,7 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); + // Issue: https://github.com/elastic/kibana/issues/165138 describe.skip('Threshold rule - DOCUMENTS_COUNT - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts index 4a8b8cb65bae..b288c4edf28a 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts @@ -30,6 +30,7 @@ export default function ({ getService }: FtrProviderContext) { let alertId: string; let startedAt: string; + // Issue: https://github.com/elastic/kibana/issues/165138 describe.skip('Threshold rule - GROUP_BY - FIRED', () => { const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; From 53501273666593de3c7dd861d8d54351894ab36a Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 29 Aug 2023 11:33:11 -0600 Subject: [PATCH 175/182] Response ops issue --- .../api_integration/test_suites/common/alerting/rules.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts index 34ece9a12921..1ad798758b45 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/alerting/rules.ts @@ -35,6 +35,7 @@ export default function ({ getService }: FtrProviderContext) { const esClient = getService('es'); const esDeleteAllIndices = getService('esDeleteAllIndices'); + // Issue: https://github.com/elastic/kibana/issues/165145 describe.skip('Alerting rules', () => { const RULE_TYPE_ID = '.es-query'; const ALERT_ACTION_INDEX = 'alert-action-es-query'; From b63c02146afe94bbb91afc6e2bd501035bdccd30 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 29 Aug 2023 18:43:00 +0100 Subject: [PATCH 176/182] chore(NA): skip burning specs that are buggy with respect to tags --- .../cypress/e2e/data_sources/create_runtime_field.cy.ts | 2 +- .../cypress/e2e/data_sources/sourcerer.cy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts index b9a514b8c221..9e12b3baf598 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts @@ -25,7 +25,7 @@ import { GET_TIMELINE_HEADER } from '../../screens/timeline'; const alertRunTimeField = 'field.name.alert.page'; const timelineRuntimeField = 'field.name.timeline'; -describe( +describe.skip( 'Create DataView runtime field', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts index ce6b6454c412..7e636f5517f6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts @@ -122,7 +122,7 @@ describe('Sourcerer', () => { cy.get(SOURCERER.saveButton).should('be.disabled'); }); - it( + it.skip( 'adds a pattern to the default index and correctly filters out auditbeat-*', { tags: '@brokenInServerless' }, () => { From 4faeae4e1046c851345c8d832efab5b21c64c898 Mon Sep 17 00:00:00 2001 From: Brad White Date: Tue, 29 Aug 2023 11:45:45 -0600 Subject: [PATCH 177/182] Add kill to FTR for safety --- packages/kbn-test/src/es/test_es_cluster.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 8857df7c41cc..580840b6b35a 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -237,6 +237,7 @@ export function createTestEsCluster< ssl: true, background: true, files, + kill: true, // likely don't need this but avoids any issues where the ESS cluster wasn't cleaned up }); } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; From 1136a48ebd02935bd7bd88310f8f9bd63a020fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= Date: Wed, 30 Aug 2023 11:02:55 +0200 Subject: [PATCH 178/182] Update alert_details_right_panel_json_tab.cy.ts --- .../expandable_flyout/alert_details_right_panel_json_tab.cy.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts index c5726980196e..01c11a671cb6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts @@ -16,8 +16,7 @@ import { getNewRule } from '../../../../objects/rule'; import { ALERTS_URL } from '../../../../urls/navigation'; import { waitForAlertsToPopulate } from '../../../../tasks/create_new_rule'; -// Failing constantly on CI -describe.skip( +describe( 'Alert details expandable flyout right panel json tab', { tags: ['@ess', '@brokenInServerless'] }, () => { From 4e75fc2af29900d446b7606cf700f13d42d0fd8a Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 30 Aug 2023 13:02:47 +0200 Subject: [PATCH 179/182] fix test --- .../e2e/data_sources/create_runtime_field.cy.ts | 2 +- .../cypress/e2e/data_sources/sourcerer.cy.ts | 6 +++--- .../alert_details_right_panel_json_tab.cy.ts | 6 +----- .../alert_details_right_panel_json_tab.ts | 15 --------------- 4 files changed, 5 insertions(+), 24 deletions(-) delete mode 100644 x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout/alert_details_right_panel_json_tab.ts diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts index 9e12b3baf598..b9a514b8c221 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/create_runtime_field.cy.ts @@ -25,7 +25,7 @@ import { GET_TIMELINE_HEADER } from '../../screens/timeline'; const alertRunTimeField = 'field.name.alert.page'; const timelineRuntimeField = 'field.name.timeline'; -describe.skip( +describe( 'Create DataView runtime field', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts index 7e636f5517f6..8113dac275b7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts @@ -45,7 +45,7 @@ const rolesToCreate = [secReadCasesAll]; const siemDataViewTitle = 'Security Default Data View'; const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*']; -describe('Sourcerer', () => { +describe('Sourcerer', { tags: ['@brokenInServerless'] }, () => { before(() => { cy.task('esArchiverResetKibana'); dataViews.forEach((dataView: string) => postDataView(dataView)); @@ -122,7 +122,7 @@ describe('Sourcerer', () => { cy.get(SOURCERER.saveButton).should('be.disabled'); }); - it.skip( + it( 'adds a pattern to the default index and correctly filters out auditbeat-*', { tags: '@brokenInServerless' }, () => { @@ -139,7 +139,7 @@ describe('Sourcerer', () => { ); }); }); -describe('Timeline scope', { tags: '@brokenInServerless' }, () => { +describe('Timeline scope', { tags: ['@brokenInServerless'] }, () => { beforeEach(() => { cy.clearLocalStorage(); login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts index 01c11a671cb6..193de317bed5 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_right_panel_json_tab.cy.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { scrollWithinDocumentDetailsExpandableFlyoutRightSection } from '../../../../tasks/expandable_flyout/alert_details_right_panel_json_tab'; import { openJsonTab } from '../../../../tasks/expandable_flyout/alert_details_right_panel'; import { expandFirstAlertExpandableFlyout } from '../../../../tasks/expandable_flyout/common'; import { DOCUMENT_DETAILS_FLYOUT_JSON_TAB_CONTENT } from '../../../../screens/expandable_flyout/alert_details_right_panel_json_tab'; @@ -31,10 +30,7 @@ describe( }); it('should display the json component', () => { - // the json component is rendered within a dom element with overflow, so Cypress isn't finding it - // this next line is a hack that vertically scrolls down to ensure Cypress finds it - scrollWithinDocumentDetailsExpandableFlyoutRightSection(0, 7000); - cy.get(DOCUMENT_DETAILS_FLYOUT_JSON_TAB_CONTENT).should('be.visible'); + cy.get(DOCUMENT_DETAILS_FLYOUT_JSON_TAB_CONTENT).should('exist'); }); } ); diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout/alert_details_right_panel_json_tab.ts b/x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout/alert_details_right_panel_json_tab.ts deleted file mode 100644 index 8affc2c7c4ce..000000000000 --- a/x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout/alert_details_right_panel_json_tab.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DOCUMENT_DETAILS_FLYOUT_RIGHT_PANEL_CONTENT } from '../../screens/expandable_flyout/alert_details_right_panel_json_tab'; - -/** - * Scroll to x-y positions within the right section of the document details expandable flyout - * // TODO revisit this as it seems very fragile: the first element found is the timeline flyout, which isn't visible but still exist in the DOM - */ -export const scrollWithinDocumentDetailsExpandableFlyoutRightSection = (x: number, y: number) => - cy.get(DOCUMENT_DETAILS_FLYOUT_RIGHT_PANEL_CONTENT).last().scrollTo(x, y); From 383a776419acf9f32c0ff1c81ad92aded880ef62 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 30 Aug 2023 18:05:01 +0200 Subject: [PATCH 180/182] split sourcerer --- .../cypress/e2e/data_sources/sourcerer.cy.ts | 141 +-------------- .../e2e/data_sources/sourcerer_timeline.ts | 167 ++++++++++++++++++ 2 files changed, 169 insertions(+), 139 deletions(-) create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer_timeline.ts diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts index 8113dac275b7..20225150d61e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts @@ -5,40 +5,30 @@ * 2.0. */ -import { - DEFAULT_ALERTS_INDEX, - DEFAULT_INDEX_PATTERN, -} from '@kbn/security-solution-plugin/common/constants'; +import { DEFAULT_INDEX_PATTERN } from '@kbn/security-solution-plugin/common/constants'; import { login, loginWithUser, visit, visitWithUser } from '../../tasks/login'; -import { HOSTS_URL, TIMELINES_URL } from '../../urls/navigation'; +import { HOSTS_URL } from '../../urls/navigation'; import { addIndexToDefault, - clickAlertCheckbox, deselectSourcererOptions, isDataViewSelection, isHostsStatValue, isKibanaDataViewOption, - isNotSourcererOption, isNotSourcererSelection, isSourcererOptions, isSourcererSelection, openAdvancedSettings, openDataViewSelection, openSourcerer, - refreshUntilAlertsIndexExists, resetSourcerer, saveSourcerer, } from '../../tasks/sourcerer'; import { postDataView } from '../../tasks/common'; -import { openTimelineUsingToggle } from '../../tasks/security_main'; import { createUsersAndRoles, secReadCasesAll, secReadCasesAllUser } from '../../tasks/privileges'; import { TOASTER } from '../../screens/configure_cases'; import { SOURCERER } from '../../screens/sourcerer'; -import { createTimeline } from '../../tasks/api_calls/timelines'; -import { getTimeline, getTimelineModifiedSourcerer } from '../../objects/timeline'; -import { closeTimeline, openTimelineById } from '../../tasks/timeline'; const usersToCreate = [secReadCasesAllUser]; const rolesToCreate = [secReadCasesAll]; @@ -139,130 +129,3 @@ describe('Sourcerer', { tags: ['@brokenInServerless'] }, () => { ); }); }); -describe('Timeline scope', { tags: ['@brokenInServerless'] }, () => { - beforeEach(() => { - cy.clearLocalStorage(); - login(); - visit(TIMELINES_URL); - }); - - it('correctly loads SIEM data view', () => { - openTimelineUsingToggle(); - openSourcerer('timeline'); - isDataViewSelection(siemDataViewTitle); - openAdvancedSettings(); - isSourcererSelection(`auditbeat-*`); - isSourcererSelection(`${DEFAULT_ALERTS_INDEX}-default`); - isSourcererOptions(DEFAULT_INDEX_PATTERN.filter((pattern) => pattern !== 'auditbeat-*')); - isNotSourcererOption(`${DEFAULT_ALERTS_INDEX}-default`); - }); - - describe('Modified badge', () => { - it('Selecting new data view does not add a modified badge', () => { - openTimelineUsingToggle(); - cy.get(SOURCERER.badgeModified).should(`not.exist`); - openSourcerer('timeline'); - cy.get(SOURCERER.badgeModifiedOption).should(`not.exist`); - openDataViewSelection(); - isKibanaDataViewOption(dataViews); - cy.get(SOURCERER.selectListDefaultOption).should(`contain`, siemDataViewTitle); - cy.get(SOURCERER.selectListOption).contains(dataViews[1]).click(); - isDataViewSelection(dataViews[1]); - saveSourcerer(); - cy.get(SOURCERER.badgeModified).should(`not.exist`); - openSourcerer('timeline'); - cy.get(SOURCERER.badgeModifiedOption).should(`not.exist`); - }); - - it('shows modified badge when index patterns change and removes when reset', () => { - openTimelineUsingToggle(); - openSourcerer('timeline'); - openDataViewSelection(); - cy.get(SOURCERER.selectListOption).contains(dataViews[1]).click(); - isDataViewSelection(dataViews[1]); - openAdvancedSettings(); - const patterns = dataViews[1].split(','); - deselectSourcererOptions([patterns[0]]); - saveSourcerer(); - cy.get(SOURCERER.badgeModified).should(`exist`); - openSourcerer('timeline'); - cy.get(SOURCERER.badgeModifiedOption).should(`exist`); - resetSourcerer(); - saveSourcerer(); - cy.get(SOURCERER.badgeModified).should(`not.exist`); - openSourcerer('timeline'); - cy.get(SOURCERER.badgeModifiedOption).should(`not.exist`); - isDataViewSelection(siemDataViewTitle); - }); - }); - describe('Alerts checkbox', () => { - before(() => { - login(); - createTimeline(getTimeline()).then((response) => - cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('timelineId') - ); - createTimeline(getTimelineModifiedSourcerer()).then((response) => - cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('auditbeatTimelineId') - ); - }); - - beforeEach(() => { - login(); - visit(TIMELINES_URL); - refreshUntilAlertsIndexExists(); - }); - - it('Modifies timeline to alerts only, and switches to different saved timeline without issue', function () { - openTimelineById(this.timelineId).then(() => { - cy.get(SOURCERER.badgeAlerts).should(`not.exist`); - cy.get(SOURCERER.badgeModified).should(`not.exist`); - openSourcerer('timeline'); - clickAlertCheckbox(); - saveSourcerer(); - cy.get(SOURCERER.badgeAlerts).should(`exist`); - cy.get(SOURCERER.badgeModified).should(`not.exist`); - closeTimeline(); - - openTimelineById(this.auditbeatTimelineId).then(() => { - cy.get(SOURCERER.badgeModified).should(`exist`); - cy.get(SOURCERER.badgeAlerts).should(`not.exist`); - openSourcerer('timeline'); - openAdvancedSettings(); - isSourcererSelection(`auditbeat-*`); - }); - }); - }); - - const defaultPatterns = [`auditbeat-*`, `${DEFAULT_ALERTS_INDEX}-default`]; - it('alerts checkbox behaves as expected', () => { - isDataViewSelection(siemDataViewTitle); - defaultPatterns.forEach((pattern) => isSourcererSelection(pattern)); - openDataViewSelection(); - cy.get(SOURCERER.selectListOption).contains(dataViews[1]).click(); - isDataViewSelection(dataViews[1]); - dataViews[1] - .split(',') - .filter((pattern) => pattern !== 'fakebeat-*' && pattern !== 'siem-read*') - .forEach((pattern) => isSourcererSelection(pattern)); - - clickAlertCheckbox(); - isNotSourcererSelection(`auditbeat-*`); - isSourcererSelection(`${DEFAULT_ALERTS_INDEX}-default`); - cy.get(SOURCERER.alertCheckbox).uncheck({ force: true }); - defaultPatterns.forEach((pattern) => isSourcererSelection(pattern)); - }); - - it('shows alerts badge when index patterns change and removes when reset', () => { - clickAlertCheckbox(); - saveSourcerer(); - cy.get(SOURCERER.badgeAlerts).should(`exist`); - openSourcerer('timeline'); - cy.get(SOURCERER.badgeAlertsOption).should(`exist`); - resetSourcerer(); - saveSourcerer(); - cy.get(SOURCERER.badgeAlerts).should(`not.exist`); - openSourcerer('timeline'); - cy.get(SOURCERER.badgeAlertsOption).should(`not.exist`); - }); - }); -}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer_timeline.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer_timeline.ts new file mode 100644 index 000000000000..1476e82421cd --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer_timeline.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + DEFAULT_ALERTS_INDEX, + DEFAULT_INDEX_PATTERN, +} from '@kbn/security-solution-plugin/common/constants'; + +import { login, visit } from '../../tasks/login'; + +import { TIMELINES_URL } from '../../urls/navigation'; +import { + clickAlertCheckbox, + deselectSourcererOptions, + isDataViewSelection, + isKibanaDataViewOption, + isNotSourcererOption, + isNotSourcererSelection, + isSourcererOptions, + isSourcererSelection, + openAdvancedSettings, + openDataViewSelection, + openSourcerer, + refreshUntilAlertsIndexExists, + resetSourcerer, + saveSourcerer, +} from '../../tasks/sourcerer'; +import { openTimelineUsingToggle } from '../../tasks/security_main'; +import { SOURCERER } from '../../screens/sourcerer'; +import { createTimeline } from '../../tasks/api_calls/timelines'; +import { getTimeline, getTimelineModifiedSourcerer } from '../../objects/timeline'; +import { closeTimeline, openTimelineById } from '../../tasks/timeline'; + +const siemDataViewTitle = 'Security Default Data View'; +const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*']; + +describe('Timeline scope', { tags: '@brokenInServerless' }, () => { + beforeEach(() => { + cy.clearLocalStorage(); + login(); + visit(TIMELINES_URL); + }); + + it('correctly loads SIEM data view', () => { + openTimelineUsingToggle(); + openSourcerer('timeline'); + isDataViewSelection(siemDataViewTitle); + openAdvancedSettings(); + isSourcererSelection(`auditbeat-*`); + isSourcererSelection(`${DEFAULT_ALERTS_INDEX}-default`); + isSourcererOptions(DEFAULT_INDEX_PATTERN.filter((pattern) => pattern !== 'auditbeat-*')); + isNotSourcererOption(`${DEFAULT_ALERTS_INDEX}-default`); + }); + + describe('Modified badge', () => { + it('Selecting new data view does not add a modified badge', () => { + openTimelineUsingToggle(); + cy.get(SOURCERER.badgeModified).should(`not.exist`); + openSourcerer('timeline'); + cy.get(SOURCERER.badgeModifiedOption).should(`not.exist`); + openDataViewSelection(); + isKibanaDataViewOption(dataViews); + cy.get(SOURCERER.selectListDefaultOption).should(`contain`, siemDataViewTitle); + cy.get(SOURCERER.selectListOption).contains(dataViews[1]).click(); + isDataViewSelection(dataViews[1]); + saveSourcerer(); + cy.get(SOURCERER.badgeModified).should(`not.exist`); + openSourcerer('timeline'); + cy.get(SOURCERER.badgeModifiedOption).should(`not.exist`); + }); + + it('shows modified badge when index patterns change and removes when reset', () => { + openTimelineUsingToggle(); + openSourcerer('timeline'); + openDataViewSelection(); + cy.get(SOURCERER.selectListOption).contains(dataViews[1]).click(); + isDataViewSelection(dataViews[1]); + openAdvancedSettings(); + const patterns = dataViews[1].split(','); + deselectSourcererOptions([patterns[0]]); + saveSourcerer(); + cy.get(SOURCERER.badgeModified).should(`exist`); + openSourcerer('timeline'); + cy.get(SOURCERER.badgeModifiedOption).should(`exist`); + resetSourcerer(); + saveSourcerer(); + cy.get(SOURCERER.badgeModified).should(`not.exist`); + openSourcerer('timeline'); + cy.get(SOURCERER.badgeModifiedOption).should(`not.exist`); + isDataViewSelection(siemDataViewTitle); + }); + }); + describe('Alerts checkbox', () => { + before(() => { + login(); + createTimeline(getTimeline()).then((response) => + cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('timelineId') + ); + createTimeline(getTimelineModifiedSourcerer()).then((response) => + cy.wrap(response.body.data.persistTimeline.timeline.savedObjectId).as('auditbeatTimelineId') + ); + }); + + beforeEach(() => { + login(); + visit(TIMELINES_URL); + refreshUntilAlertsIndexExists(); + }); + + it('Modifies timeline to alerts only, and switches to different saved timeline without issue', function () { + openTimelineById(this.timelineId).then(() => { + cy.get(SOURCERER.badgeAlerts).should(`not.exist`); + cy.get(SOURCERER.badgeModified).should(`not.exist`); + openSourcerer('timeline'); + clickAlertCheckbox(); + saveSourcerer(); + cy.get(SOURCERER.badgeAlerts).should(`exist`); + cy.get(SOURCERER.badgeModified).should(`not.exist`); + closeTimeline(); + + openTimelineById(this.auditbeatTimelineId).then(() => { + cy.get(SOURCERER.badgeModified).should(`exist`); + cy.get(SOURCERER.badgeAlerts).should(`not.exist`); + openSourcerer('timeline'); + openAdvancedSettings(); + isSourcererSelection(`auditbeat-*`); + }); + }); + }); + + const defaultPatterns = [`auditbeat-*`, `${DEFAULT_ALERTS_INDEX}-default`]; + it('alerts checkbox behaves as expected', () => { + isDataViewSelection(siemDataViewTitle); + defaultPatterns.forEach((pattern) => isSourcererSelection(pattern)); + openDataViewSelection(); + cy.get(SOURCERER.selectListOption).contains(dataViews[1]).click(); + isDataViewSelection(dataViews[1]); + dataViews[1] + .split(',') + .filter((pattern) => pattern !== 'fakebeat-*' && pattern !== 'siem-read*') + .forEach((pattern) => isSourcererSelection(pattern)); + + clickAlertCheckbox(); + isNotSourcererSelection(`auditbeat-*`); + isSourcererSelection(`${DEFAULT_ALERTS_INDEX}-default`); + cy.get(SOURCERER.alertCheckbox).uncheck({ force: true }); + defaultPatterns.forEach((pattern) => isSourcererSelection(pattern)); + }); + + it('shows alerts badge when index patterns change and removes when reset', () => { + clickAlertCheckbox(); + saveSourcerer(); + cy.get(SOURCERER.badgeAlerts).should(`exist`); + openSourcerer('timeline'); + cy.get(SOURCERER.badgeAlertsOption).should(`exist`); + resetSourcerer(); + saveSourcerer(); + cy.get(SOURCERER.badgeAlerts).should(`not.exist`); + openSourcerer('timeline'); + cy.get(SOURCERER.badgeAlertsOption).should(`not.exist`); + }); + }); +}); From d790522a830764a052f9d5fbd5571b7ca0f523e0 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 30 Aug 2023 17:30:01 +0100 Subject: [PATCH 181/182] fix(NA): remove extra log declaration from parallel.ts --- .../security_solution/scripts/run_cypress/parallel.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index a2b60c5bbbad..281ce2e1ac5a 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -171,11 +171,6 @@ export const cli = () => { _.pull(fleetServerPorts, fleetServerPort); }; - const log = new ToolingLog({ - level: 'info', - writeTo: process.stdout, - }); - await pMap( files, async (filePath) => { From f1fee628bfb35ace805176d4b20bba6712ee4412 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 30 Aug 2023 19:55:39 +0200 Subject: [PATCH 182/182] fix --- .../scripts/run_cypress/parallel.ts | 29 ++++++++++--------- .../cypress/e2e/data_sources/sourcerer.cy.ts | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 281ce2e1ac5a..45b48a5d428e 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -63,34 +63,35 @@ const retrieveIntegrations = (integrationsPaths: string[]) => { export const cli = () => { run( async () => { - const { argv } = yargs(process.argv.slice(2)).coerce('env', (arg: string) => - arg.split(',').reduce((acc, curr) => { - const [key, value] = curr.split('='); - if (key === 'burn') { - acc[key] = parseInt(value, 10); - } else { - acc[key] = value; - } - return acc; - }, {} as Record) - ); + const { argv } = yargs(process.argv.slice(2)) + .coerce('spec', (arg) => (_.isArray(arg) ? [_.last(arg)] : [arg])) + .coerce('env', (arg: string) => + arg.split(',').reduce((acc, curr) => { + const [key, value] = curr.split('='); + if (key === 'burn') { + acc[key] = parseInt(value, 10); + } else { + acc[key] = value; + } + return acc; + }, {} as Record) + ); const isOpen = argv._[0] === 'open'; const cypressConfigFilePath = require.resolve( `../../${_.isArray(argv.configFile) ? _.last(argv.configFile) : argv.configFile}` ) as string; const cypressConfigFile = await import(cypressConfigFilePath); - const spec: string | undefined = argv?.spec as string; const grepSpecPattern = grep({ ...cypressConfigFile, - specPattern: spec ?? cypressConfigFile.e2e.specPattern, + specPattern: argv.spec ?? cypressConfigFile.e2e.specPattern, excludeSpecPattern: [], }).specPattern; let files = retrieveIntegrations( _.isArray(grepSpecPattern) ? grepSpecPattern - : globby.sync(spec ? [spec] : cypressConfigFile.e2e.specPattern) + : globby.sync(argv.spec ?? cypressConfigFile.e2e.specPattern) ); if (argv.changedSpecsOnly) { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts index 20225150d61e..77ce5b56d30a 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/data_sources/sourcerer.cy.ts @@ -35,7 +35,7 @@ const rolesToCreate = [secReadCasesAll]; const siemDataViewTitle = 'Security Default Data View'; const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*']; -describe('Sourcerer', { tags: ['@brokenInServerless'] }, () => { +describe('Sourcerer', () => { before(() => { cy.task('esArchiverResetKibana'); dataViews.forEach((dataView: string) => postDataView(dataView));