From 89e2425539ab8e909d89f43314274a05a2e8f43e Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 25 Oct 2023 17:51:12 -0400 Subject: [PATCH 01/72] Added `getMultipassVmCountNotice()` and `generateVmName()` to VM services --- .../scripts/endpoint/common/vm_services.ts | 40 +++++++++++++++++++ .../endpoint_agent_runner/elastic_endpoint.ts | 28 ++----------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index ca0a83d2cdd41..f7ed38d87d200 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -8,9 +8,13 @@ import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import chalk from 'chalk'; +import { userInfo } from 'os'; +import { BaseDataGenerator } from '../../../common/endpoint/data_generators/base_data_generator'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import type { HostVm, HostVmExecResponse, SupportedVmManager } from './types'; +const baseGenerator = new BaseDataGenerator(); + interface BaseVmCreateOptions { name: string; /** Number of CPUs */ @@ -119,3 +123,39 @@ export const createMultipassHostVmClient = ( unmount, }; }; + +/** + * Generates a unique Virtual Machine name using the current user's `username` + * @param identifier + */ +export const generateVmName = (identifier: string = baseGenerator.randomUser()): string => { + return `${userInfo().username.toLowerCase().replaceAll('.', '-')}-${identifier + .toLowerCase() + .replace('.', '-')}-${Math.random().toString().substring(2, 6)}`; +}; + +/** + * Checks if the count of VM running under Multipass is greater than the `threshold` passed on + * input and if so, it will return a message indicate so. Useful to remind users of the amount of + * VM currently running. + * @param threshold + */ +export const getMultipassVmCountNotice = async (threshold: number = 1): Promise => { + const response = await execa.command(`multipass list --format=json`); + + const output: { list: Array<{ ipv4: string; name: string; release: string; state: string }> } = + JSON.parse(response.stdout); + + if (output.list.length > threshold) { + return `----------------------------------------------------------------- +${chalk.red('NOTE:')} ${chalk.bold( + chalk.cyan(`You currently have ${chalk.red(output.list.length)} VMs running.`) + )} Remember to delete those + no longer being used. + View running VMs: ${chalk.bold('multipass list')} + ----------------------------------------------------------------- +`; + } + + return ''; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts index 0c07013f8c7b7..738206943bb86 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts @@ -6,8 +6,8 @@ */ import { userInfo } from 'os'; -import execa from 'execa'; import chalk from 'chalk'; +import { getMultipassVmCountNotice } from '../common/vm_services'; import { createAndEnrollEndpointHost } from '../common/endpoint_host_services'; import { addEndpointIntegrationToAgentPolicy, @@ -68,7 +68,9 @@ export const enrollEndpointHost = async (): Promise => { Elastic Agent Version: ${version} Shell access: ${chalk.bold(`multipass shell ${vmName}`)} - Delete VM: ${chalk.bold(`multipass delete -p ${vmName}${await getVmCountNotice()}`)} + Delete VM: ${chalk.bold( + `multipass delete -p ${vmName}${await getMultipassVmCountNotice()}` + )} `); } } catch (error) { @@ -90,25 +92,3 @@ const getOrCreateAgentPolicyId = async (): Promise => { return agentPolicy.id; }; - -const getVmCountNotice = async (threshold: number = 1): Promise => { - const response = await execa.command(`multipass list --format=json`); - - const output: { list: Array<{ ipv4: string; name: string; release: string; state: string }> } = - JSON.parse(response.stdout); - - if (output.list.length > threshold) { - return ` - ------------------------------------------------------------------ -${chalk.red('NOTE:')} ${chalk.bold( - `You currently have ${output.list.length} VMs running.` - )} Remember to delete those - no longer being used. - View running VMs: ${chalk.bold('multipass list')} - ----------------------------------------------------------------- -`; - } - - return ''; -}; From d3ad8ff01a2ba53b64756d5ac53dd268aa487359 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 25 Oct 2023 17:52:30 -0400 Subject: [PATCH 02/72] Create VM and enroll it with Fleet for the agentless integrations --- .../scripts/endpoint/common/fleet_services.ts | 20 ++-- .../endpoint/sentinelone_host/index.ts | 92 ++++++++++++++----- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 80825c84fb195..bd5035aafe793 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -72,6 +72,9 @@ import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fl const fleetGenerator = new FleetAgentGenerator(); const CURRENT_USERNAME = userInfo().username.toLowerCase(); +const DEFAULT_AGENT_POLICY_NAME = `${CURRENT_USERNAME} test policy`; +/** A Fleet agent policy that includes integrations that don't actually require an agent to run on a host. Example: SenttinelOne */ +export const DEFAULT_AGENTLESS_INTEGRATIONS_AGENT_POLICY_NAME = `${CURRENT_USERNAME} - agentless integrations`; export const checkInFleetAgent = async ( esClient: Client, @@ -578,7 +581,10 @@ interface EnrollHostVmWithFleetOptions { } /** - * Installs the Elastic agent on the provided Host VM and enrolls with it Fleet + * Installs the Elastic agent on the provided Host VM and enrolls with it Fleet. + * + * NOTE: this method assumes that FLeet-Server is already setup and running. + * * @param hostVm * @param kbnClient * @param log @@ -669,6 +675,7 @@ export const enrollHostVmWithFleet = async ({ interface GetOrCreateDefaultAgentPolicyOptions { kbnClient: KbnClient; log: ToolingLog; + policyName?: string; } /** @@ -676,14 +683,15 @@ interface GetOrCreateDefaultAgentPolicyOptions { * policy already exists, then it will be reused. * @param kbnClient * @param log + * @param policyName */ export const getOrCreateDefaultAgentPolicy = async ({ kbnClient, log, + policyName = DEFAULT_AGENT_POLICY_NAME, }: GetOrCreateDefaultAgentPolicyOptions): Promise => { - const agentPolicyName = `${CURRENT_USERNAME} test policy`; const existingPolicy = await fetchAgentPolicyList(kbnClient, { - kuery: `${AGENT_POLICY_SAVED_OBJECT_TYPE}.name: "${agentPolicyName}"`, + kuery: `${AGENT_POLICY_SAVED_OBJECT_TYPE}.name: "${policyName}"`, }); if (existingPolicy.items[0]) { @@ -696,8 +704,8 @@ export const getOrCreateDefaultAgentPolicy = async ({ log.info(`Creating new default test/dev Fleet agent policy`); const newAgentPolicyData: CreateAgentPolicyRequest['body'] = { - name: agentPolicyName, - description: `Policy created by security solution tooling`, + name: policyName, + description: `Policy created by security solution tooling: ${__filename}`, namespace: 'default', monitoring_enabled: ['logs', 'metrics'], }; @@ -793,7 +801,7 @@ export const addSentinelOneIntegrationToAgentPolicy = async ({ agentPolicyId, consoleUrl, apiToken, - integrationPolicyName = `SentinelOne policy (${Math.random().toString().substring(3)})`, + integrationPolicyName = `SentinelOne policy (${Math.random().toString().substring(2, 6)})`, force = false, }: AddSentinelOneIntegrationToAgentPolicyOptions): Promise => { // If `force` is `false and agent policy already has a SentinelOne integration, exit here diff --git a/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts index a5aec7a52aec9..d2f01cbb8abfd 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts @@ -7,27 +7,35 @@ import type { RunFn } from '@kbn/dev-cli-runner'; import { run } from '@kbn/dev-cli-runner'; -import { userInfo } from 'os'; import { ok } from 'assert'; +import { + isFleetServerRunning, + startFleetServer, +} from '../common/fleet_server/fleet_server_services'; +import type { HostVm } from '../common/types'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import { addSentinelOneIntegrationToAgentPolicy, + DEFAULT_AGENTLESS_INTEGRATIONS_AGENT_POLICY_NAME, + enrollHostVmWithFleet, + fetchAgentPolicy, getOrCreateDefaultAgentPolicy, } from '../common/fleet_services'; import { installSentinelOneAgent, S1Client } from './common'; -import { createVm } from '../common/vm_services'; -import { createRuntimeServices } from '../common/stack_services'; +import { createVm, generateVmName, getMultipassVmCountNotice } from '../common/vm_services'; +import { createKbnClient } from '../common/stack_services'; export const cli = async () => { // TODO:PT add support for CPU, Disk and Memory input args return run(runCli, { - description: - 'Creates a new VM and runs both SentinelOne agent and elastic agent with the SentinelOne integration', + description: `Sets up the kibana system so that SentinelOne hosts data can be be streamed to Elasticsearch. +It will first setup a host VM that runs the SentinelOne agent on it. It will then +also setup a second VM (if necessary) that runs Elastic Agent along with the SentinelOne integration +policy (an agent-less integration).`, flags: { string: [ 'kibanaUrl', - 'elasticUrl', 'username', 'password', 'version', @@ -36,29 +44,28 @@ export const cli = async () => { 's1ApiToken', 'vmName', ], - boolean: ['force'], + boolean: ['forceFleetServer'], default: { kibanaUrl: 'http://127.0.0.1:5601', - elasticUrl: 'http://127.0.0.1:9200', username: 'elastic', password: 'changeme', - version: '', policy: '', - force: false, }, help: ` --s1Url Required. The base URL for SentinelOne management console. Ex: https://usea1-partners.sentinelone.net (valid as of Oct. 2023) --s1ApiToken Required. The API token for SentinelOne - --vmName Optional. The name for the VM + --vmName Optional. The name for the VM. + Default: [current login user name]-sentinelone-[unique number] --policy Optional. The UUID of the Fleet Agent Policy that should be used to setup the SentinelOne Integration Default: re-uses existing dev policy (if found) or creates a new one + --forceFleetServer Optional. If fleet server should be started/configured even if it seems + like it is already setup. --username Optional. User name to be used for auth against elasticsearch and kibana (Default: elastic). --password Optional. Password associated with the username (Default: changeme) --kibanaUrl Optional. The url to Kibana (Default: http://127.0.0.1:5601) - --elasticUrl Optional. The url to Elasticsearch (Default: http://127.0.0.1:9200) `, }, }); @@ -68,10 +75,10 @@ const runCli: RunFn = async ({ log, flags }) => { const username = flags.username as string; const password = flags.password as string; const kibanaUrl = flags.kibanaUrl as string; - const elasticUrl = flags.elasticUrl as string; const s1Url = flags.s1Url as string; const s1ApiToken = flags.s1ApiToken as string; const policy = flags.policy as string; + const forceFleetServer = flags.forceFleetServer as boolean; createToolingLogger.defaultLogLevel = flags.verbose ? 'verbose' @@ -88,18 +95,13 @@ const runCli: RunFn = async ({ log, flags }) => { ok(s1Url, getRequiredArgMessage('s1Url')); ok(s1ApiToken, getRequiredArgMessage('s1ApiToken')); - const vmName = - (flags.vmName as string) || - `${userInfo().username.toLowerCase().replaceAll('.', '-')}-sentinelone-${Math.random() - .toString() - .substring(2, 6)}`; + const vmName = (flags.vmName as string) || generateVmName('sentinelone'); const s1Client = new S1Client({ url: s1Url, apiToken: s1ApiToken, log }); - const { kbnClient } = await createRuntimeServices({ - kibanaUrl, - elasticsearchUrl: elasticUrl, + const kbnClient = createKbnClient({ + log, + url: kibanaUrl, username, password, - log, }); const hostVm = await createVm({ @@ -116,7 +118,17 @@ const runCli: RunFn = async ({ log, flags }) => { s1Client, }); - const agentPolicyId = policy || (await getOrCreateDefaultAgentPolicy({ kbnClient, log })).id; + const { + id: agentPolicyId, + agents = 0, + name: agentPolicyName, + } = policy + ? await fetchAgentPolicy(kbnClient, policy) + : await getOrCreateDefaultAgentPolicy({ + kbnClient, + log, + policyName: DEFAULT_AGENTLESS_INTEGRATIONS_AGENT_POLICY_NAME, + }); await addSentinelOneIntegrationToAgentPolicy({ kbnClient, @@ -126,10 +138,42 @@ const runCli: RunFn = async ({ log, flags }) => { apiToken: s1ApiToken, }); + let agentPolicyVm: HostVm | undefined; + + // If no agents are running against the given Agent policy for agentless integrations, then add one now + if (!agents) { + log.info(`Creating VM and enrolling it with Fleet using policy [${agentPolicyName}]`); + + agentPolicyVm = await createVm({ + type: 'multipass', + name: generateVmName('agentless-integrations'), + }); + + if (forceFleetServer || !(await isFleetServerRunning(kbnClient, log))) { + await startFleetServer({ + kbnClient, + logger: log, + force: forceFleetServer, + }); + } + + await enrollHostVmWithFleet({ + hostVm: agentPolicyVm, + kbnClient, + log, + agentPolicyId, + }); + } else { + log.info( + `No host VM created for Fleet agent policy [${agentPolicyName}]. It already show to have [${agents}] enrolled` + ); + } + log.info(`Done! ${hostVm.info()} - +${agentPolicyVm ? `${agentPolicyVm.info()}\n` : ''} +${await getMultipassVmCountNotice(2)} SentinelOne Agent Status: ${s1Info.status} `); From 0a58c6185df12c766a581a321c4652cb61a922ea Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 10:16:10 -0400 Subject: [PATCH 03/72] General improvements + added `setDefaultLogLevelFromCliFlags()` to `createToolingLogger` --- .../common/endpoint/data_loaders/utils.ts | 24 +++++++++++++++++-- .../endpoint/sentinelone_host/index.ts | 23 +++++++----------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts index c27fb5fc7154d..0586aca1d0c76 100644 --- a/x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts +++ b/x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts @@ -8,6 +8,7 @@ import { mergeWith } from 'lodash'; import type { ToolingLogTextWriterConfig } from '@kbn/tooling-log'; import { ToolingLog } from '@kbn/tooling-log'; +import type { Flags } from '@kbn/dev-cli-runner'; export const RETRYABLE_TRANSIENT_ERRORS: Readonly> = [ 'no_shard_available_action_exception', @@ -117,12 +118,20 @@ interface CreateLoggerInterface { * on input. */ defaultLogLevel: ToolingLogTextWriterConfig['level']; + + /** + * Set the default logging level based on the flag arguments provide to a CLI script that runs + * via `@kbn/dev-cli-runner` + * @param flags + */ + setDefaultLogLevelFromCliFlags: (flags: Flags) => void; } /** * Creates an instance of `ToolingLog` that outputs to `stdout`. - * The default log `level` for all instances can be set by setting the function's `defaultLogLevel`. - * Log level can also be explicitly set on input. + * The default log `level` for all instances can be set by setting the function's `defaultLogLevel` + * property. Default logging level can also be set from CLI scripts that use the `@kbn/dev-cli-runner` + * by calling the `setDefaultLogLevelFromCliFlags(flags)` and passing in the `flags` property. * * @param level * @@ -137,3 +146,14 @@ export const createToolingLogger: CreateLoggerInterface = (level): ToolingLog => }); }; createToolingLogger.defaultLogLevel = 'info'; +createToolingLogger.setDefaultLogLevelFromCliFlags = (flags) => { + createToolingLogger.defaultLogLevel = flags.verbose + ? 'verbose' + : flags.debug + ? 'debug' + : flags.silent + ? 'silent' + : flags.quiet + ? 'error' + : 'info'; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts index d2f01cbb8abfd..3fad8bf0223bb 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/index.ts @@ -30,9 +30,11 @@ export const cli = async () => { return run(runCli, { description: `Sets up the kibana system so that SentinelOne hosts data can be be streamed to Elasticsearch. -It will first setup a host VM that runs the SentinelOne agent on it. It will then -also setup a second VM (if necessary) that runs Elastic Agent along with the SentinelOne integration -policy (an agent-less integration).`, +It will first setup a host VM that runs the SentinelOne agent on it. This VM will ensure that data is being +created in SentinelOne. +It will then also setup a second VM (if necessary) that runs Elastic Agent along with the SentinelOne integration +policy (an agent-less integration) - this is the process that then connects to the SentinelOne management +console and pushes the data to Elasticsearch.`, flags: { string: [ 'kibanaUrl', @@ -79,19 +81,10 @@ const runCli: RunFn = async ({ log, flags }) => { const s1ApiToken = flags.s1ApiToken as string; const policy = flags.policy as string; const forceFleetServer = flags.forceFleetServer as boolean; - - createToolingLogger.defaultLogLevel = flags.verbose - ? 'verbose' - : flags.debug - ? 'debug' - : flags.silent - ? 'silent' - : flags.quiet - ? 'error' - : 'info'; - const getRequiredArgMessage = (argName: string) => `${argName} argument is required`; + createToolingLogger.setDefaultLogLevelFromCliFlags(flags); + ok(s1Url, getRequiredArgMessage('s1Url')); ok(s1ApiToken, getRequiredArgMessage('s1ApiToken')); @@ -165,7 +158,7 @@ const runCli: RunFn = async ({ log, flags }) => { }); } else { log.info( - `No host VM created for Fleet agent policy [${agentPolicyName}]. It already show to have [${agents}] enrolled` + `No host VM created for Fleet agent policy [${agentPolicyName}]. It already shows to have [${agents}] enrolled` ); } From dbd1e2d0b1f251dd9d15b37608d6e842b1fb162b Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 10:40:53 -0400 Subject: [PATCH 04/72] Refactor Run Endpoint Agent to use common fleet server service --- .../fleet_server/fleet_server_services.ts | 5 +- .../endpoint_agent_runner/fleet_server.ts | 504 +----------------- 2 files changed, 17 insertions(+), 492 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 79dcdd458d359..12ac61effbc9f 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -95,7 +95,7 @@ interface StartFleetServerOptions { } interface StartedFleetServer extends StartedServer { - /** The policy id that the fleet-server agent is running with */ + /** The Fleet Agent policy id that the fleet-server agent is running with */ policyId: string; } @@ -110,8 +110,6 @@ export const startFleetServer = async ({ logger.info(`Starting Fleet Server and connecting it to Kibana`); return logger.indent(4, async () => { - const isServerless = await isServerlessKibanaFlavor(kbnClient); - // Check if fleet already running if `force` is false if (!force && (await isFleetServerRunning(kbnClient))) { throw new Error( @@ -119,6 +117,7 @@ export const startFleetServer = async ({ ); } + const isServerless = await isServerlessKibanaFlavor(kbnClient); const policyId = policy || !isServerless ? await getOrCreateFleetServerAgentPolicyId(kbnClient, logger) : ''; const serviceToken = isServerless ? '' : await generateFleetServiceToken(kbnClient, logger); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts index ec13f2f6ff1b1..07d9dd8522052 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts @@ -5,502 +5,28 @@ * 2.0. */ -import { - CA_TRUSTED_FINGERPRINT, - FLEET_SERVER_CERT_PATH, - FLEET_SERVER_KEY_PATH, - fleetServerDevServiceAccount, -} from '@kbn/dev-utils'; -import type { - AgentPolicy, - CreateAgentPolicyResponse, - GetPackagePoliciesResponse, - Output, - PackagePolicy, -} from '@kbn/fleet-plugin/common'; -import { - AGENT_POLICY_API_ROUTES, - API_VERSIONS, - APP_API_ROUTES, - FLEET_SERVER_PACKAGE, - PACKAGE_POLICY_API_ROUTES, - PACKAGE_POLICY_SAVED_OBJECT_TYPE, -} from '@kbn/fleet-plugin/common'; -import type { - FleetServerHost, - GenerateServiceTokenResponse, - GetOneOutputResponse, - GetOutputsResponse, - PutOutputRequest, -} from '@kbn/fleet-plugin/common/types'; -import { - fleetServerHostsRoutesService, - outputRoutesService, -} from '@kbn/fleet-plugin/common/services'; -import execa from 'execa'; -import type { - PostFleetServerHostsRequest, - PostFleetServerHostsResponse, -} from '@kbn/fleet-plugin/common/types/rest_spec/fleet_server_hosts'; -import chalk from 'chalk'; -import { maybeCreateDockerNetwork, SERVERLESS_NODES, verifyDockerInstalled } from '@kbn/es'; -import { FLEET_SERVER_CUSTOM_CONFIG } from '../common/fleet_server/fleet_server_services'; -import { isServerlessKibanaFlavor } from '../common/stack_services'; -import type { FormattedAxiosError } from '../common/format_axios_error'; -import { catchAxiosErrorFormatAndThrow } from '../common/format_axios_error'; -import { isLocalhost } from '../common/is_localhost'; -import { dump } from './utils'; -import { fetchFleetServerUrl, waitForHostToEnroll } from '../common/fleet_services'; +import { startFleetServer } from '../common/fleet_server/fleet_server_services'; import { getRuntimeServices } from './runtime'; export const runFleetServerIfNeeded = async (): Promise< { fleetServerContainerId: string; fleetServerAgentPolicyId: string | undefined } | undefined > => { - let fleetServerContainerId; - let fleetServerAgentPolicyId; - let serviceToken; - - const { - log, - kibana: { isLocalhost: isKibanaOnLocalhost }, - kbnClient, - } = getRuntimeServices(); - - log.info(`Setting up fleet server (if necessary)`); - log.indent(4); - const isServerless = await isServerlessKibanaFlavor(kbnClient); - - await verifyDockerInstalled(log); - await maybeCreateDockerNetwork(log); - - try { - if (isServerless) { - fleetServerContainerId = await startFleetServerStandAloneWithDocker(); - } else { - fleetServerAgentPolicyId = await getOrCreateFleetServerAgentPolicyId(); - serviceToken = await generateFleetServiceToken(); - if (isKibanaOnLocalhost) { - await configureFleetIfNeeded(); - } - fleetServerContainerId = await startFleetServerWithDocker({ - policyId: fleetServerAgentPolicyId, - serviceToken, - }); - } - } catch (error) { - log.error(dump(error)); - log.indent(-4); - throw error; - } - - log.indent(-4); - - return { fleetServerContainerId, fleetServerAgentPolicyId }; -}; - -const getFleetServerPackagePolicy = async (): Promise => { - const { kbnClient } = getRuntimeServices(); - - return kbnClient - .request({ - method: 'GET', - path: PACKAGE_POLICY_API_ROUTES.LIST_PATTERN, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - query: { - perPage: 1, - kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: "${FLEET_SERVER_PACKAGE}"`, - }, - }) - .then((response) => response.data.items[0]); -}; - -const getOrCreateFleetServerAgentPolicyId = async (): Promise => { const { log, kbnClient } = getRuntimeServices(); - const existingFleetServerIntegrationPolicy = await getFleetServerPackagePolicy(); - - if (existingFleetServerIntegrationPolicy) { - log.verbose( - `Found existing Fleet Server Policy: ${JSON.stringify( - existingFleetServerIntegrationPolicy, - null, - 2 - )}` - ); - log.info( - `Using existing Fleet Server agent policy id: ${existingFleetServerIntegrationPolicy.policy_id}` - ); - - return existingFleetServerIntegrationPolicy.policy_id; - } - - log.info(`Creating new Fleet Server policy`); - - const createdFleetServerPolicy: AgentPolicy = await kbnClient - .request({ - method: 'POST', - path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: { - name: `Fleet Server policy (${Math.random().toString(32).substring(2)})`, - description: `Created by CLI Tool via: ${__filename}`, - namespace: 'default', - monitoring_enabled: ['logs', 'metrics'], - // This will ensure the Fleet Server integration policy - // is also created and added to the agent policy - has_fleet_server: true, - }, - }) - .then((response) => response.data.item); - - log.indent(4); - log.info( - `Agent Policy created: ${createdFleetServerPolicy.name} (${createdFleetServerPolicy.id})` - ); - log.verbose(createdFleetServerPolicy); - log.indent(-4); - - return createdFleetServerPolicy.id; -}; - -const generateFleetServiceToken = async (): Promise => { - const { kbnClient, log } = getRuntimeServices(); - - const serviceToken: string = await kbnClient - .request({ - method: 'POST', - path: APP_API_ROUTES.GENERATE_SERVICE_TOKEN_PATTERN, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: {}, - }) - .then((response) => response.data.value); - - log.info(`New service token created.`); - - return serviceToken; -}; + // Runs under CI should force fleet server to be installed because the CI process first + // setups up Fleet server URL in kibana and then expects the server to be started. Forcing + // install here will ensure the fleet server is setup even if it appears to already be setup + // for kibana. + const forceInstall = Boolean(process.env.CI); -export const startFleetServerWithDocker = async ({ - policyId, - serviceToken, -}: { - policyId: string; - serviceToken: string; -}) => { - let containerId; - const { - log, - localhostRealIp, - elastic: { url: elasticUrl, isLocalhost: isElasticOnLocalhost }, - fleetServer: { port: fleetServerPort }, + const startedFleetServer = await startFleetServer({ kbnClient, - options: { version }, - } = getRuntimeServices(); - - log.info(`Starting a new fleet server using Docker`); - log.indent(4); - - const esURL = new URL(elasticUrl); - const containerName = `dev-fleet-server.${fleetServerPort}`; - let esUrlWithRealIp: string = elasticUrl; - - if (isElasticOnLocalhost) { - esURL.hostname = localhostRealIp; - esUrlWithRealIp = esURL.toString(); - } - - try { - const dockerArgs = [ - 'run', - '--restart', - 'no', - '--net', - 'elastic', - '--add-host', - 'host.docker.internal:host-gateway', - '--rm', - '--detach', - '--name', - containerName, - // The container's hostname will appear in Fleet when the agent enrolls - '--hostname', - containerName, - '--env', - 'FLEET_SERVER_ENABLE=1', - '--env', - `FLEET_SERVER_ELASTICSEARCH_HOST=${esUrlWithRealIp}`, - '--env', - `FLEET_SERVER_SERVICE_TOKEN=${serviceToken}`, - '--env', - `FLEET_SERVER_POLICY=${policyId}`, - '--publish', - `${fleetServerPort}:8220`, - `docker.elastic.co/beats/elastic-agent:${version}`, - ]; - - await execa('docker', ['kill', containerName]) - .then(() => { - log.verbose( - `Killed an existing container with name [${containerName}]. New one will be started.` - ); - }) - .catch((error) => { - log.verbose(`Attempt to kill currently running fleet-server container (if any) with name [${containerName}] was unsuccessful: - ${error} -(This is ok if one was not running already)`); - }); - - await addFleetServerHostToFleetSettings(`https://${localhostRealIp}:${fleetServerPort}`); - - log.verbose(`docker arguments:\n${dockerArgs.join(' ')}`); - - containerId = (await execa('docker', dockerArgs)).stdout; - - const fleetServerAgent = await waitForHostToEnroll(kbnClient, containerName, 120000); - - log.verbose(`Fleet server enrolled agent:\n${JSON.stringify(fleetServerAgent, null, 2)}`); - - log.info(`Done. Fleet Server is running and connected to Fleet. - Container Name: ${containerName} - Container Id: ${containerId} - - View running output: ${chalk.bold(`docker attach ---sig-proxy=false ${containerName}`)} - Shell access: ${chalk.bold(`docker exec -it ${containerName} /bin/bash`)} - Kill container: ${chalk.bold(`docker kill ${containerId}`)} -`); - } catch (error) { - log.error(dump(error)); - log.indent(-4); - throw error; - } - - log.indent(-4); - - return containerId; -}; - -export const startFleetServerStandAloneWithDocker = async () => { - let containerId; - const { - log, - elastic: { url: elasticUrl }, - fleetServer: { port: fleetServerPort }, - } = getRuntimeServices(); - - log.info(`Starting a new fleet server using Docker`); - log.indent(4); - const esURL = new URL(elasticUrl); - - esURL.hostname = SERVERLESS_NODES[0].name; - - const esUrlWithRealIp = esURL.toString(); - - const containerName = `dev-fleet-server.${fleetServerPort}`; - try { - const dockerArgs = [ - 'run', - '--restart', - 'no', - '--net', - 'elastic', - '--add-host', - 'host.docker.internal:host-gateway', - '--rm', - '--detach', - '--name', - containerName, - // The container's hostname will appear in Fleet when the agent enrolls - '--hostname', - containerName, - '--volume', - `${FLEET_SERVER_CERT_PATH}:/fleet-server.crt`, - '--volume', - `${FLEET_SERVER_KEY_PATH}:/fleet-server.key`, - '--env', - 'FLEET_SERVER_CERT=/fleet-server.crt', - '--env', - 'FLEET_SERVER_CERT_KEY=/fleet-server.key', - '--env', - `ELASTICSEARCH_HOSTS=${esUrlWithRealIp}`, - '--env', - `ELASTICSEARCH_SERVICE_TOKEN=${fleetServerDevServiceAccount.token}`, - '--env', - `ELASTICSEARCH_CA_TRUSTED_FINGERPRINT=${CA_TRUSTED_FINGERPRINT}`, - '--volume', - `${FLEET_SERVER_CUSTOM_CONFIG}:/etc/fleet-server.yml:ro`, - '--publish', - `${fleetServerPort}:8220`, - `docker.elastic.co/observability-ci/fleet-server:latest`, - ]; - - await execa('docker', ['kill', containerName]) - .then(() => { - log.verbose( - `Killed an existing container with name [${containerName}]. New one will be started.` - ); - }) - .catch((error) => { - log.verbose(`Attempt to kill currently running fleet-server container (if any) with name [${containerName}] was unsuccessful: - ${error} -(This is ok if one was not running already)`); - }); - - log.verbose(`docker arguments:\n${dockerArgs.join(' ')}`); - - containerId = (await execa('docker', dockerArgs)).stdout; - - log.info(`Done. Fleet Server Stand Alone is running and connected to Fleet. - Container Name: ${containerName} - Container Id: ${containerId} - - View running output: ${chalk.bold(`docker attach ---sig-proxy=false ${containerName}`)} - Shell access: ${chalk.bold(`docker exec -it ${containerName} /bin/bash`)} - Kill container: ${chalk.bold(`docker kill ${containerId}`)} -`); - } catch (error) { - log.error(dump(error)); - log.indent(-4); - throw error; - } - - log.indent(-4); - - return containerId; -}; - -const configureFleetIfNeeded = async () => { - const { log, kbnClient, localhostRealIp } = getRuntimeServices(); - - log.info('Checking if Fleet needs to be configured'); - log.indent(4); - - try { - // make sure that all ES hostnames are using localhost real IP - const fleetOutputs = await kbnClient - .request({ - method: 'GET', - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - path: outputRoutesService.getListPath(), - }) - .then((response) => response.data); - - for (const { id, ...output } of fleetOutputs.items) { - if (output.type === 'elasticsearch') { - if (output.hosts) { - let needsUpdating = false; - const updatedHosts: Output['hosts'] = []; - - for (const host of output.hosts) { - const hostURL = new URL(host); - - if (isLocalhost(hostURL.hostname)) { - needsUpdating = true; - hostURL.hostname = localhostRealIp; - updatedHosts.push(hostURL.toString()); - - log.verbose( - `Fleet Settings for Elasticsearch Output [Name: ${ - output.name - } (id: ${id})]: Host [${host}] updated to [${hostURL.toString()}]` - ); - } else { - updatedHosts.push(host); - } - } - - if (needsUpdating) { - const update: PutOutputRequest['body'] = { - ...(output as PutOutputRequest['body']), // cast needed to quite TS - looks like the types for Output in fleet differ a bit between create/update - hosts: updatedHosts, - }; - - log.info(`Updating Fleet Settings for Output [${output.name} (${id})]`); - - await kbnClient - .request({ - method: 'PUT', - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - path: outputRoutesService.getUpdatePath(id), - body: update, - }) - .catch(catchAxiosErrorFormatAndThrow); - } - } - } - } - } catch (error) { - log.error(dump(error)); - log.indent(-4); - throw error; - } - - log.indent(-4); -}; - -const addFleetServerHostToFleetSettings = async ( - fleetServerHostUrl: string -): Promise => { - const { kbnClient, log } = getRuntimeServices(); - - log.info(`Updating Fleet with new fleet server host: ${fleetServerHostUrl}`); - log.indent(4); - - try { - const exitingFleetServerHostUrl = await fetchFleetServerUrl(kbnClient); - - const newFleetHostEntry: PostFleetServerHostsRequest['body'] = { - name: `Dev fleet server running on localhost`, - host_urls: [fleetServerHostUrl], - is_default: !exitingFleetServerHostUrl, - }; - - const { item } = await kbnClient - .request({ - method: 'POST', - path: fleetServerHostsRoutesService.getCreatePath(), - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: newFleetHostEntry, - }) - .catch(catchAxiosErrorFormatAndThrow) - .catch((error: FormattedAxiosError) => { - if ( - error.response.status === 403 && - ((error.response?.data?.message as string) ?? '').includes('disabled') - ) { - log.error(`Update failed with [403: ${error.response.data.message}]. - -${chalk.red('Are you running this utility against a Serverless project?')} -If so, the following entry should be added to your local -'config/serverless.[project_type].dev.yml' (ex. 'serverless.security.dev.yml'): - -${chalk.bold(chalk.cyan('xpack.fleet.internal.fleetServerStandalone: false'))} - -`); - } - - throw error; - }) - .then((response) => response.data); - - log.verbose(item); - log.indent(-4); - - return item; - } catch (error) { - log.error(dump(error)); - log.indent(-4); - throw error; - } + logger: log, + force: forceInstall, + }); + + return { + fleetServerContainerId: startedFleetServer.id, + fleetServerAgentPolicyId: startedFleetServer.policyId, + }; }; From a71307853dc5e7819aad522e9016bfe13e5f782d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 12:55:21 -0400 Subject: [PATCH 05/72] Added support for Vagrant VMs to `createVm()` --- src/dev/precommit_hook/casing_check_config.js | 2 +- .../vagrant}/Vagrantfile | 4 + .../scripts/endpoint/common/vm_services.ts | 150 ++++++++++++++++-- 3 files changed, 146 insertions(+), 10 deletions(-) rename x-pack/plugins/security_solution/scripts/endpoint/{endpoint_agent_runner => common/vagrant}/Vagrantfile (75%) diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 93eba1bb171b2..acf4748055589 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -44,7 +44,7 @@ export const IGNORE_FILE_GLOBS = [ 'packages/kbn-test/jest-preset.js', 'packages/kbn-test/*/jest-preset.js', 'test/package/Vagrantfile', - 'x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/Vagrantfile', + 'x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile', '**/test/**/fixtures/**/*', // Required to match the name in the docs.elastic.dev repo. diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/Vagrantfile b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile similarity index 75% rename from x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/Vagrantfile rename to x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile index 42080eab2984e..92c4cba8d2dd4 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/Vagrantfile +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile @@ -1,3 +1,7 @@ +# ------------------------------------------------------------------------------------ +# Vagrant setup for running Elastic agent on the created VM.0 +# This setup is mostly used for CI runs, since multipass is not used in that env. +# ------------------------------------------------------------------------------------ hostname = ENV["VMNAME"] || 'ubuntu' cachedAgentSource = ENV["CACHED_AGENT_SOURCE"] || '' cachedAgentFilename = ENV["CACHED_AGENT_FILENAME"] || '' diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index f7ed38d87d200..621c739447ae2 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -9,13 +9,15 @@ import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import chalk from 'chalk'; import { userInfo } from 'os'; +import { resolve, dirname } from 'path'; +import type { DownloadedAgentInfo } from './agent_downloads_service'; import { BaseDataGenerator } from '../../../common/endpoint/data_generators/base_data_generator'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import type { HostVm, HostVmExecResponse, SupportedVmManager } from './types'; const baseGenerator = new BaseDataGenerator(); -interface BaseVmCreateOptions { +export interface BaseVmCreateOptions { name: string; /** Number of CPUs */ cpus?: number; @@ -25,24 +27,21 @@ interface BaseVmCreateOptions { memory?: string; } -interface CreateVmOptions extends BaseVmCreateOptions { - /** The type of VM manager to use when creating the VM host */ - type: SupportedVmManager; - log?: ToolingLog; -} +type CreateVmOptions = CreateMultipassVmOptions | CreateVagrantVmOptions; /** * Creates a new VM */ -export const createVm = async ({ type, ...options }: CreateVmOptions): Promise => { - if (type === 'multipass') { +export const createVm = async (options: CreateVmOptions): Promise => { + if (options.type === 'multipass') { return createMultipassVm(options); } - throw new Error(`VM type ${type} not yet supported`); + return createVagrantVm(options); }; interface CreateMultipassVmOptions extends BaseVmCreateOptions { + type: SupportedVmManager & 'multipass'; name: string; log?: ToolingLog; } @@ -159,3 +158,136 @@ ${chalk.red('NOTE:')} ${chalk.bold( return ''; }; + +interface CreateVagrantVmOptions extends BaseVmCreateOptions { + type: SupportedVmManager & 'vagrant'; + + name: string; + /** + * The downloaded agent information. The Agent file will be uploaded to the Vagrant VM and + * made available under the default login home directory (`~/agent-filename`) + */ + agentDownload: DownloadedAgentInfo; + /** + * The path to the Vagrantfile to use to provision the VM. Defaults to Vagrantfile under: + * `x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile` + */ + vagrantFile?: string; + log?: ToolingLog; +} + +/** + * Creates a new VM using `vagrant` + */ +const createVagrantVm = async ({ + name, + log = createToolingLogger(), + agentDownload: { fullFilePath: agentFullFilePath, filename: agentFileName }, + vagrantFile = resolve('../endpoint_agent_runner/Vagrantfile'), + memory, + cpus, + disk, +}: CreateVagrantVmOptions): Promise => { + log.debug(`Using Vagrantfile: ${vagrantFile}`); + + const VAGRANT_CWD = dirname(vagrantFile); + + // Destroy the VM running (if any) with the provided vagrant file before re-creating it + try { + await execa.command(`vagrant destroy -f`, { + env: { + VAGRANT_CWD, + }, + // Only `pipe` STDERR to parent process + stdio: ['inherit', 'inherit', 'pipe'], + }); + // eslint-disable-next-line no-empty + } catch (e) {} + + if (memory || cpus || disk) { + log.warning( + `cpu, memory and disk options ignored for creation of vm via Vagrant. These should be defined in the Vagrantfile` + ); + } + + try { + const vagrantUpResponse = ( + await execa.command(`vagrant up`, { + env: { + VAGRANT_DISABLE_VBOXSYMLINKCREATE: '1', + VAGRANT_CWD, + VMNAME: name, + CACHED_AGENT_SOURCE: agentFullFilePath, + CACHED_AGENT_FILENAME: agentFileName, + }, + // Only `pipe` STDERR to parent process + stdio: ['inherit', 'inherit', 'pipe'], + }) + ).stdout; + + log.debug(`Vagrant up command response: `, vagrantUpResponse); + } catch (e) { + log.error(e); + throw e; + } + + return createVagrantHostVmClient(name, log); +}; + +/** + * Creates a generic interface (`HotVm`) for interacting with a VM creatd by Vagrant + * @param name + * @param log + */ +export const createVagrantHostVmClient = ( + name: string, + log: ToolingLog = createToolingLogger() +): HostVm => { + const exec = async (command: string): Promise => { + const execResponse = await execa.command(`vagrant ssh ${name} --command="${command}"`); + + log.verbose(execResponse); + + return { + stdout: execResponse.stdout, + stderr: execResponse.stderr, + exitCode: execResponse.exitCode, + }; + }; + + const destroy = async (): Promise => { + const destroyResponse = await execa.command(`vagrant destroy ${name} -f`, { + // Only `pipe` STDERR to parent process + stdio: ['inherit', 'inherit', 'pipe'], + }); + + log.verbose(`VM [${name}] was destroyed successfully`, destroyResponse); + }; + + const info = () => { + return `VM created using Vagrant. + VM Name: ${name} + + Shell access: ${chalk.cyan(`vagrant ssh ${name}`)} + Delete VM: ${chalk.cyan(`vagrant destroy ${name} -f`)} +`; + }; + + const unmount = async (_: string) => { + throw new Error('VM action `unmount`` not currently supported for vagrant'); + }; + + const mount = async (_: string, __: string) => { + throw new Error('VM action `mount` not currently supported for vagrant'); + }; + + return { + type: 'vagrant', + name, + exec, + destroy, + info, + mount, + unmount, + }; +}; From 2922b326904929bb066b9aec63c2598f5397c0fe Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 13:09:11 -0400 Subject: [PATCH 06/72] Refactor to use `createVm()` and its "client" creators for all types of VMs --- .../endpoint/common/endpoint_host_services.ts | 232 +++++------------- .../scripts/endpoint/common/types.ts | 2 + .../scripts/endpoint/common/vm_services.ts | 20 ++ .../endpoint_agent_runner/elastic_endpoint.ts | 31 +-- 4 files changed, 87 insertions(+), 198 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index 303ae81a9197a..61e4135ef3616 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -10,6 +10,9 @@ import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import assert from 'assert'; +import type { HostVm } from './types'; +import type { BaseVmCreateOptions } from './vm_services'; +import { createMultipassHostVmClient, createVagrantHostVmClient, createVm } from './vm_services'; import type { DownloadedAgentInfo } from './agent_downloads_service'; import { cleanupDownloads, downloadAndStoreAgent } from './agent_downloads_service'; import { @@ -23,7 +26,7 @@ import { export const VAGRANT_CWD = `${__dirname}/../endpoint_agent_runner/`; export interface CreateAndEnrollEndpointHostOptions - extends Pick { + extends Pick { kbnClient: KbnClient; log: ToolingLog; /** The fleet Agent Policy ID to use for enrolling the agent */ @@ -41,6 +44,7 @@ export interface CreateAndEnrollEndpointHostOptions export interface CreateAndEnrollEndpointHostResponse { hostname: string; agentId: string; + hostVm: HostVm; } /** @@ -82,15 +86,18 @@ export const createAndEnrollEndpointHost = async ({ return { url }; }); - const [vm, fleetServerUrl, enrollmentToken] = await Promise.all([ + const [hostVm, fleetServerUrl, enrollmentToken] = await Promise.all([ process.env.CI - ? createVagrantVm({ - vmName, + ? createVm({ + type: 'vagrant', + name: vmName, log, - cachedAgentDownload: agentDownload.cache as DownloadedAgentInfo, + agentDownload: agentDownload.cache as DownloadedAgentInfo, }) - : createMultipassVm({ - vmName, + : createVm({ + type: 'multipass', + log, + name: vmName, disk, cpus, memory, @@ -101,16 +108,12 @@ export const createAndEnrollEndpointHost = async ({ fetchAgentPolicyEnrollmentKey(kbnClient, agentPolicyId), ]); - if (!process.env.CI) { - log.verbose(await execa('multipass', ['info', vm.vmName])); - } - // Some validations before we proceed assert(agentDownload.url, 'Missing agent download URL'); assert(fleetServerUrl, 'Fleet server URL not set'); assert(enrollmentToken, `No enrollment token for agent policy id [${agentPolicyId}]`); - log.verbose(`Enrolling host [${vm.vmName}] + log.verbose(`Enrolling host [${hostVm.name}] with fleet-server [${fleetServerUrl}] using enrollment token [${enrollmentToken}]`); @@ -121,7 +124,7 @@ export const createAndEnrollEndpointHost = async ({ agentDownloadUrl: agentDownload.url, cachedAgentDownload: agentDownload.cache, enrollmentToken, - vmName: vm.vmName, + hostVm, }); await cacheCleanupPromise.then((results) => { @@ -135,8 +138,9 @@ ${results.deleted.join('\n')} }); return { - hostname: vm.vmName, + hostname: hostVm.name, agentId, + hostVm, }; }; @@ -155,101 +159,18 @@ export const destroyEndpointHost = async ( ]); }; -interface CreateVmResponse { - vmName: string; -} - -interface CreateVagrantVmOptions { - vmName: string; - cachedAgentDownload: DownloadedAgentInfo; - log: ToolingLog; -} - -/** - * Creates a new VM using `vagrant` - */ -const createVagrantVm = async ({ - vmName, - cachedAgentDownload, - log, -}: CreateVagrantVmOptions): Promise => { - try { - await execa.command(`vagrant destroy -f`, { - env: { - VAGRANT_CWD, - }, - // Only `pipe` STDERR to parent process - stdio: ['inherit', 'inherit', 'pipe'], - }); - // eslint-disable-next-line no-empty - } catch (e) {} - - try { - await execa.command(`vagrant up`, { - env: { - VAGRANT_DISABLE_VBOXSYMLINKCREATE: '1', - VAGRANT_CWD, - VMNAME: vmName, - CACHED_AGENT_SOURCE: cachedAgentDownload.fullFilePath, - CACHED_AGENT_FILENAME: cachedAgentDownload.filename, - }, - // Only `pipe` STDERR to parent process - stdio: ['inherit', 'inherit', 'pipe'], - }); - } catch (e) { - log.error(e); - throw e; - } - - return { - vmName, - }; -}; - -interface CreateMultipassVmOptions { - vmName: string; - /** Number of CPUs */ - cpus?: number; - /** Disk size */ - disk?: string; - /** Amount of memory */ - memory?: string; -} - -/** - * Creates a new VM using `multipass` - */ -const createMultipassVm = async ({ - vmName, - disk = '8G', - cpus = 1, - memory = '1G', -}: CreateMultipassVmOptions): Promise => { - await execa.command( - `multipass launch --name ${vmName} --disk ${disk} --cpus ${cpus} --memory ${memory}` - ); - - return { - vmName, - }; -}; - export const deleteMultipassVm = async (vmName: string): Promise => { - if (process.env.CI) { - await execa.command(`vagrant destroy -f`, { - env: { - VAGRANT_CWD, - }, - }); - } else { - await execa.command(`multipass delete -p ${vmName}`); - } + const hostVm = process.env.CI + ? createVagrantHostVmClient(vmName) + : createMultipassHostVmClient(vmName); + + await hostVm.destroy(); }; interface EnrollHostWithFleetOptions { kbnClient: KbnClient; log: ToolingLog; - vmName: string; + hostVm: HostVm; agentDownloadUrl: string; cachedAgentDownload?: DownloadedAgentInfo; fleetServerUrl: string; @@ -259,7 +180,7 @@ interface EnrollHostWithFleetOptions { const enrollHostWithFleet = async ({ kbnClient, log, - vmName, + hostVm, fleetServerUrl, agentDownloadUrl, cachedAgentDownload, @@ -267,27 +188,28 @@ const enrollHostWithFleet = async ({ }: EnrollHostWithFleetOptions): Promise<{ agentId: string }> => { const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1); const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, ''); + const vmName = hostVm.name; - if (cachedAgentDownload) { - log.verbose( - `Installing agent on host using cached download from [${cachedAgentDownload.fullFilePath}]` - ); + // For multipass VMs, we need to get the Agent archive into the VM and extract it + // (Vagrant VMs already has it in the VM created) + if (hostVm.type === 'multipass') { + const downloadsHostVmMountedDir = '~/_agent_downloads'; - if (!process.env.CI) { - // mount local folder on VM - await execa.command( - `multipass mount ${cachedAgentDownload.directory} ${vmName}:~/_agent_downloads` + if (cachedAgentDownload) { + log.debug( + `Installing agent on host using cached download from [${cachedAgentDownload.fullFilePath}]` ); - await execa.command( - `multipass exec ${vmName} -- tar -zxf _agent_downloads/${cachedAgentDownload.filename}` - ); - await execa.command(`multipass unmount ${vmName}:~/_agent_downloads`); - } - } else { - log.verbose(`downloading and installing agent from URL [${agentDownloadUrl}]`); - if (!process.env.CI) { - // download into VM + log.debug(`mounting [${cachedAgentDownload.directory}] to [${downloadsHostVmMountedDir}]`); + await hostVm.mount(cachedAgentDownload.directory, downloadsHostVmMountedDir); + + log.debug(`Extracting ${cachedAgentDownload.filename}`); + await hostVm.exec(`tar -zxf _agent_downloads/${cachedAgentDownload.filename}`); + + log.debug(`un-mounting [${downloadsHostVmMountedDir}]`); + } else { + log.debug(`Downloading Agent to host VM: ${agentDownloadUrl}`); + await execa.command( `multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}` ); @@ -296,7 +218,7 @@ const enrollHostWithFleet = async ({ } } - const agentInstallArguments = [ + const agentInstallCommand = [ 'sudo', './elastic-agent', @@ -312,34 +234,14 @@ const enrollHostWithFleet = async ({ '--enrollment-token', enrollmentToken, - ]; + ].join(' '); log.info(`Enrolling elastic agent with Fleet`); - if (process.env.CI) { - log.verbose(`Command: vagrant ${agentInstallArguments.join(' ')}`); - - await execa(`vagrant`, ['ssh', '--', `cd ${vmDirName} && ${agentInstallArguments.join(' ')}`], { - env: { - VAGRANT_CWD, - }, - // Only `pipe` STDERR to parent process - stdio: ['inherit', 'inherit', 'pipe'], - }); - } else { - log.verbose(`Command: multipass ${agentInstallArguments.join(' ')}`); - - await execa(`multipass`, [ - 'exec', - vmName, - '--working-directory', - `/home/ubuntu/${vmDirName}`, - - '--', - ...agentInstallArguments, - ]); - } - log.info(`Waiting for Agent to check-in with Fleet`); + log.debug(`Agent enroll command:\n${agentInstallCommand}`); + + await hostVm.exec(`cd ${vmDirName} && ${agentInstallCommand}`); + log.info(`Waiting for Agent to check-in with Fleet`); const agent = await waitForHostToEnroll(kbnClient, vmName, 8 * 60 * 1000); log.info(`Agent enrolled with Fleet, status: `, agent.status); @@ -349,32 +251,18 @@ const enrollHostWithFleet = async ({ }; }; -export async function getEndpointHosts(): Promise< - Array<{ name: string; state: string; ipv4: string; image: string }> -> { - const output = await execa('multipass', ['list', '--format', 'json']); - return JSON.parse(output.stdout).list; -} +export async function stopEndpointHost(hostName: string): Promise { + const hostVm = process.env.CI + ? createVagrantHostVmClient(hostName) + : createMultipassHostVmClient(hostName); -export function stopEndpointHost(hostName: string) { - if (process.env.CI) { - return execa('vagrant', ['suspend'], { - env: { - VAGRANT_CWD, - VMNAME: hostName, - }, - }); - } - return execa('multipass', ['stop', hostName]); + await hostVm.stop(); } -export function startEndpointHost(hostName: string) { - if (process.env.CI) { - return execa('vagrant', ['up'], { - env: { - VAGRANT_CWD, - }, - }); - } - return execa('multipass', ['start', hostName]); +export async function startEndpointHost(hostName: string): Promise { + const hostVm = process.env.CI + ? createVagrantHostVmClient(hostName) + : createMultipassHostVmClient(hostName); + + await hostVm.start(); } diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts index 44bdcff328401..77fc34ea8e4d4 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts @@ -16,6 +16,8 @@ export interface HostVm { unmount: (hostVmDir: string) => Promise; destroy: () => Promise; info: () => string; + stop: () => void; + start: () => void; } export type SupportedVmManager = 'multipass' | 'vagrant'; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 621c739447ae2..8d7c51ba79024 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -112,6 +112,14 @@ export const createMultipassHostVmClient = ( }; }; + const start = async () => { + await execa.command(`multipass start ${name}`); + }; + + const stop = async () => { + await execa.command(`multipass stop ${name}`); + }; + return { type: 'multipass', name, @@ -120,6 +128,8 @@ export const createMultipassHostVmClient = ( info, mount, unmount, + start, + stop, }; }; @@ -281,6 +291,14 @@ export const createVagrantHostVmClient = ( throw new Error('VM action `mount` not currently supported for vagrant'); }; + const start = async () => { + await execa.command(`vagrant up ${name}`); + }; + + const stop = async () => { + await execa.command(`vagrant suspend ${name}`); + }; + return { type: 'vagrant', name, @@ -289,5 +307,7 @@ export const createVagrantHostVmClient = ( info, mount, unmount, + start, + stop, }; }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts index 738206943bb86..e569580eb7357 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts @@ -5,9 +5,7 @@ * 2.0. */ -import { userInfo } from 'os'; -import chalk from 'chalk'; -import { getMultipassVmCountNotice } from '../common/vm_services'; +import { generateVmName } from '../common/vm_services'; import { createAndEnrollEndpointHost } from '../common/endpoint_host_services'; import { addEndpointIntegrationToAgentPolicy, @@ -18,6 +16,7 @@ import { dump } from './utils'; export const enrollEndpointHost = async (): Promise => { let vmName; + const { log, kbnClient, @@ -28,8 +27,6 @@ export const enrollEndpointHost = async (): Promise => { log.indent(4); try { - const uniqueId = Math.random().toString().substring(2, 6); - const username = userInfo().username.toLowerCase().replaceAll('.', '-'); // Multipass doesn't like periods in username const policyId: string = policy || (await getOrCreateAgentPolicyId()); if (!policyId) { @@ -40,11 +37,11 @@ export const enrollEndpointHost = async (): Promise => { throw new Error(`No 'version' specified`); } - vmName = `${username}-dev-${uniqueId}`; + vmName = generateVmName('dev'); log.info(`Creating VM named: ${vmName}`); - await createAndEnrollEndpointHost({ + const { hostVm } = await createAndEnrollEndpointHost({ kbnClient, log, hostname: vmName, @@ -54,25 +51,7 @@ export const enrollEndpointHost = async (): Promise => { disk: '8G', }); - if (process.env.CI) { - log.info(`VM created using Vagrant. - VM Name: ${vmName} - Elastic Agent Version: ${version} - - Shell access: ${chalk.bold(`vagrant ssh ${vmName}`)} - Delete VM: ${chalk.bold(`vagrant destroy ${vmName} -f`)} - `); - } else { - log.info(`VM created using Multipass. - VM Name: ${vmName} - Elastic Agent Version: ${version} - - Shell access: ${chalk.bold(`multipass shell ${vmName}`)} - Delete VM: ${chalk.bold( - `multipass delete -p ${vmName}${await getMultipassVmCountNotice()}` - )} - `); - } + log.info(hostVm.info()); } catch (error) { log.error(dump(error)); log.indent(-4); From 94a9e95c1a723fec9709c94aaef808ff1f5a7c39 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 16:43:20 -0400 Subject: [PATCH 07/72] Make the fleet-server updated entry in fleet the default --- .../endpoint/common/fleet_server/fleet_server_services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 12ac61effbc9f..38450f6afff56 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -493,7 +493,7 @@ const addFleetServerHostToFleetSettings = async ( const newFleetHostEntry: PostFleetServerHostsRequest['body'] = { name: `Dev fleet server running on localhost`, host_urls: [fleetServerHostUrl], - is_default: !exitingFleetServerHostList.total, + is_default: true, }; const { item } = await kbnClient From 2b482059b7bc130f609ee5047ec705e3c5358f7d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 16:44:01 -0400 Subject: [PATCH 08/72] Enhanced Agent download service to auto-cleanup after each download --- .../endpoint/common/agent_downloads_service.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts index 6f392006df8bf..34f473a854460 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts @@ -10,6 +10,7 @@ import { join } from 'path'; import fs from 'fs'; import nodeFetch from 'node-fetch'; import { finished } from 'stream/promises'; +import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import { SettingsStorage } from './settings_storage'; export interface DownloadedAgentInfo { @@ -40,6 +41,7 @@ class AgentDownloadStorage extends SettingsStorage private downloadsFolderExists = false; private readonly downloadsDirName = 'agent_download_storage'; private readonly downloadsDirFullPath: string; + private readonly log = createToolingLogger(); constructor() { super('agent_download_storage_settings.json', { @@ -57,6 +59,7 @@ class AgentDownloadStorage extends SettingsStorage if (!this.downloadsFolderExists) { await mkdir(this.downloadsDirFullPath, { recursive: true }); + this.log.debug(`Created directory [this.downloadsDirFullPath] for cached agent downloads`); this.downloadsFolderExists = true; } } @@ -74,6 +77,8 @@ class AgentDownloadStorage extends SettingsStorage } public async downloadAndStore(agentDownloadUrl: string): Promise { + this.log.debug(`Downloading and storing: ${agentDownloadUrl}`); + // TODO: should we add "retry" attempts to file downloads? await this.ensureExists(); @@ -82,6 +87,7 @@ class AgentDownloadStorage extends SettingsStorage // If download is already present on disk, then just return that info. No need to re-download it if (fs.existsSync(newDownloadInfo.fullFilePath)) { + this.log.debug(`Download already cached at [${newDownloadInfo.fullFilePath}]`); return newDownloadInfo; } @@ -104,10 +110,14 @@ class AgentDownloadStorage extends SettingsStorage throw e; } + await this.cleanupDownloads(); + return newDownloadInfo; } public async cleanupDownloads(): Promise<{ deleted: string[] }> { + this.log.debug(`Performing cleanup of cached Agent downlaods`); + const settings = await this.get(); const maxAgeDate = new Date(); const response: { deleted: string[] } = { deleted: [] }; @@ -139,6 +149,9 @@ class AgentDownloadStorage extends SettingsStorage await Promise.allSettled(deleteFilePromises); + this.log.debug(`Deleted [${response.deleted.length}] file(s)`); + this.log.verbose(`files deleted:\n`, response.deleted.join('\n')); + return response; } } From b7b75db7ba5f82fea265238f28cf6545884ea4f0 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 16:56:32 -0400 Subject: [PATCH 09/72] Refactor endpoint host services to use common utilities and remove duplicate code --- .../endpoint/common/endpoint_host_services.ts | 189 +++--------------- .../scripts/endpoint/common/fleet_services.ts | 11 +- .../endpoint_agent_runner/fleet_server.ts | 25 ++- 3 files changed, 50 insertions(+), 175 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index 61e4135ef3616..288d422663a07 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -8,20 +8,11 @@ import { kibanaPackageJson } from '@kbn/repo-info'; import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; -import execa from 'execa'; -import assert from 'assert'; import type { HostVm } from './types'; import type { BaseVmCreateOptions } from './vm_services'; import { createMultipassHostVmClient, createVagrantHostVmClient, createVm } from './vm_services'; -import type { DownloadedAgentInfo } from './agent_downloads_service'; -import { cleanupDownloads, downloadAndStoreAgent } from './agent_downloads_service'; -import { - fetchAgentPolicyEnrollmentKey, - fetchFleetServerUrl, - getAgentDownloadUrl, - unEnrollFleetAgent, - waitForHostToEnroll, -} from './fleet_services'; +import { downloadAndStoreAgent } from './agent_downloads_service'; +import { enrollHostVmWithFleet, getAgentDownloadUrl, unEnrollFleetAgent } from './fleet_services'; export const VAGRANT_CWD = `${__dirname}/../endpoint_agent_runner/`; @@ -62,79 +53,39 @@ export const createAndEnrollEndpointHost = async ({ useClosestVersionMatch = false, useCache = true, }: CreateAndEnrollEndpointHostOptions): Promise => { - let cacheCleanupPromise: ReturnType = Promise.resolve({ - deleted: [], - }); - + const isRunningInCI = Boolean(process.env.CI); const vmName = hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`; + const { url: agentUrl } = await getAgentDownloadUrl(version, useClosestVersionMatch, log); + const agentDownload = isRunningInCI ? await downloadAndStoreAgent(agentUrl) : undefined; - const agentDownload = await getAgentDownloadUrl(version, useClosestVersionMatch, log).then<{ - url: string; - cache?: DownloadedAgentInfo; - }>(({ url }) => { - if (useCache) { - cacheCleanupPromise = cleanupDownloads(); - - return downloadAndStoreAgent(url).then((cache) => { - return { - url, - cache, - }; + const hostVm = process.env.CI + ? await createVm({ + type: 'vagrant', + name: vmName, + log, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + agentDownload: agentDownload!, + disk, + cpus, + memory, + }) + : await createVm({ + type: 'multipass', + log, + name: vmName, + disk, + cpus, + memory, }); - } - - return { url }; - }); - - const [hostVm, fleetServerUrl, enrollmentToken] = await Promise.all([ - process.env.CI - ? createVm({ - type: 'vagrant', - name: vmName, - log, - agentDownload: agentDownload.cache as DownloadedAgentInfo, - }) - : createVm({ - type: 'multipass', - log, - name: vmName, - disk, - cpus, - memory, - }), - - fetchFleetServerUrl(kbnClient), - - fetchAgentPolicyEnrollmentKey(kbnClient, agentPolicyId), - ]); - - // Some validations before we proceed - assert(agentDownload.url, 'Missing agent download URL'); - assert(fleetServerUrl, 'Fleet server URL not set'); - assert(enrollmentToken, `No enrollment token for agent policy id [${agentPolicyId}]`); - log.verbose(`Enrolling host [${hostVm.name}] - with fleet-server [${fleetServerUrl}] - using enrollment token [${enrollmentToken}]`); - - const { agentId } = await enrollHostWithFleet({ + const { id: agentId } = await enrollHostVmWithFleet({ kbnClient, log, - fleetServerUrl, - agentDownloadUrl: agentDownload.url, - cachedAgentDownload: agentDownload.cache, - enrollmentToken, hostVm, - }); - - await cacheCleanupPromise.then((results) => { - if (results.deleted.length > 0) { - log.verbose(`Agent Downloads cache directory was cleaned up and the following ${ - results.deleted.length - } were deleted: -${results.deleted.join('\n')} -`); - } + agentPolicyId, + version, + closestVersionMatch: useClosestVersionMatch, + useAgentCache: useCache, }); return { @@ -167,90 +118,6 @@ export const deleteMultipassVm = async (vmName: string): Promise => { await hostVm.destroy(); }; -interface EnrollHostWithFleetOptions { - kbnClient: KbnClient; - log: ToolingLog; - hostVm: HostVm; - agentDownloadUrl: string; - cachedAgentDownload?: DownloadedAgentInfo; - fleetServerUrl: string; - enrollmentToken: string; -} - -const enrollHostWithFleet = async ({ - kbnClient, - log, - hostVm, - fleetServerUrl, - agentDownloadUrl, - cachedAgentDownload, - enrollmentToken, -}: EnrollHostWithFleetOptions): Promise<{ agentId: string }> => { - const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1); - const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, ''); - const vmName = hostVm.name; - - // For multipass VMs, we need to get the Agent archive into the VM and extract it - // (Vagrant VMs already has it in the VM created) - if (hostVm.type === 'multipass') { - const downloadsHostVmMountedDir = '~/_agent_downloads'; - - if (cachedAgentDownload) { - log.debug( - `Installing agent on host using cached download from [${cachedAgentDownload.fullFilePath}]` - ); - - log.debug(`mounting [${cachedAgentDownload.directory}] to [${downloadsHostVmMountedDir}]`); - await hostVm.mount(cachedAgentDownload.directory, downloadsHostVmMountedDir); - - log.debug(`Extracting ${cachedAgentDownload.filename}`); - await hostVm.exec(`tar -zxf _agent_downloads/${cachedAgentDownload.filename}`); - - log.debug(`un-mounting [${downloadsHostVmMountedDir}]`); - } else { - log.debug(`Downloading Agent to host VM: ${agentDownloadUrl}`); - - await execa.command( - `multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}` - ); - await execa.command(`multipass exec ${vmName} -- tar -zxf ${agentDownloadedFile}`); - await execa.command(`multipass exec ${vmName} -- rm -f ${agentDownloadedFile}`); - } - } - - const agentInstallCommand = [ - 'sudo', - - './elastic-agent', - - 'install', - - '--insecure', - - '--force', - - '--url', - fleetServerUrl, - - '--enrollment-token', - enrollmentToken, - ].join(' '); - - log.info(`Enrolling elastic agent with Fleet`); - log.debug(`Agent enroll command:\n${agentInstallCommand}`); - - await hostVm.exec(`cd ${vmDirName} && ${agentInstallCommand}`); - - log.info(`Waiting for Agent to check-in with Fleet`); - const agent = await waitForHostToEnroll(kbnClient, vmName, 8 * 60 * 1000); - - log.info(`Agent enrolled with Fleet, status: `, agent.status); - - return { - agentId: agent.id, - }; -}; - export async function stopEndpointHost(hostName: string): Promise { const hostVm = process.env.CI ? createVagrantHostVmClient(hostName) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index bd5035aafe793..605cf6919318d 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -56,6 +56,7 @@ import nodeFetch from 'node-fetch'; import semver from 'semver'; import axios from 'axios'; import { userInfo } from 'os'; +import { isFleetServerRunning } from './fleet_server/fleet_server_services'; import { getEndpointPackageInfo } from '../../../common/endpoint/utils/package'; import type { DownloadAndStoreAgentResponse } from './agent_downloads_service'; import { downloadAndStoreAgent } from './agent_downloads_service'; @@ -606,6 +607,10 @@ export const enrollHostVmWithFleet = async ({ }: EnrollHostVmWithFleetOptions): Promise => { log.info(`Enrolling host VM [${hostVm.name}] with Fleet`); + if (!(await isFleetServerRunning(kbnClient))) { + throw new Error(`Fleet server does not seem to be running on this instance of kibana!`); + } + const agentVersion = version || (await getAgentVersionMatchingCurrentStack(kbnClient)); const agentUrlInfo = await getAgentDownloadUrl(agentVersion, closestVersionMatch, log); @@ -667,9 +672,7 @@ export const enrollHostVmWithFleet = async ({ await hostVm.exec(agentEnrollCommand); log.info(`Waiting for Agent to check-in with Fleet`); - const agent = await waitForHostToEnroll(kbnClient, hostVm.name, timeoutMs); - - return agent; + return waitForHostToEnroll(kbnClient, hostVm.name, timeoutMs); }; interface GetOrCreateDefaultAgentPolicyOptions { @@ -710,7 +713,7 @@ export const getOrCreateDefaultAgentPolicy = async ({ monitoring_enabled: ['logs', 'metrics'], }; - const newAgentPolicy = kbnClient + const newAgentPolicy = await kbnClient .request({ path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN, headers: { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts index 07d9dd8522052..d4ad9a11290a4 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts @@ -5,7 +5,10 @@ * 2.0. */ -import { startFleetServer } from '../common/fleet_server/fleet_server_services'; +import { + isFleetServerRunning, + startFleetServer, +} from '../common/fleet_server/fleet_server_services'; import { getRuntimeServices } from './runtime'; export const runFleetServerIfNeeded = async (): Promise< @@ -19,14 +22,16 @@ export const runFleetServerIfNeeded = async (): Promise< // for kibana. const forceInstall = Boolean(process.env.CI); - const startedFleetServer = await startFleetServer({ - kbnClient, - logger: log, - force: forceInstall, - }); + if (forceInstall || !(await isFleetServerRunning(kbnClient))) { + const startedFleetServer = await startFleetServer({ + kbnClient, + logger: log, + force: forceInstall, + }); - return { - fleetServerContainerId: startedFleetServer.id, - fleetServerAgentPolicyId: startedFleetServer.policyId, - }; + return { + fleetServerContainerId: startedFleetServer.id, + fleetServerAgentPolicyId: startedFleetServer.policyId, + }; + } }; From 9d9a585bf8bf7dff24fe45881913ded20be9864b Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 17:47:46 -0400 Subject: [PATCH 10/72] use common `startFleetServer()` from endpoint cypress suite --- .../cypress/support/data_loaders.ts | 25 ++++++++----------- .../endpoint/common/endpoint_host_services.ts | 3 ++- .../fleet_server/fleet_server_services.ts | 2 +- .../endpoint_agent_runner/fleet_server.ts | 5 ++++ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index fcead968801af..9808dac925966 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -11,14 +11,14 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common'; import execa from 'execa'; import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; +import type { StartedFleetServer } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; +import { startFleetServer } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; import type { KibanaKnownUserAccounts } from '../common/constants'; import { KIBANA_KNOWN_DEFAULT_ACCOUNTS } from '../common/constants'; import type { EndpointSecurityRoleNames } from '../../../../scripts/endpoint/common/roles_users'; import { SECURITY_SERVERLESS_ROLE_NAMES } from '../../../../scripts/endpoint/common/roles_users'; import type { LoadedRoleAndUser } from '../../../../scripts/endpoint/common/role_and_user_loader'; import { EndpointSecurityTestRolesLoader } from '../../../../scripts/endpoint/common/role_and_user_loader'; -import { startRuntimeServices } from '../../../../scripts/endpoint/endpoint_agent_runner/runtime'; -import { runFleetServerIfNeeded } from '../../../../scripts/endpoint/endpoint_agent_runner/fleet_server'; import { sendEndpointActionResponse, sendFleetActionResponse, @@ -290,7 +290,7 @@ export const dataLoadersForRealEndpoints = ( on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions ): void => { - let fleetServerContainerId: string | undefined; + let fleetSrv: StartedFleetServer | undefined; const stackServicesPromise = createRuntimeServices({ kibanaUrl: config.env.KIBANA_URL, @@ -304,23 +304,20 @@ export const dataLoadersForRealEndpoints = ( }); on('before:run', async () => { - await startRuntimeServices({ - kibanaUrl: config.env.KIBANA_URL, - elasticUrl: config.env.ELASTICSEARCH_URL, - fleetServerUrl: config.env.FLEET_SERVER_URL, - username: config.env.KIBANA_USERNAME, - password: config.env.KIBANA_PASSWORD, - asSuperuser: true, + const { kbnClient, log } = await stackServicesPromise; + + fleetSrv = await startFleetServer({ + kbnClient, + logger: log, + force: true, }); - const data = await runFleetServerIfNeeded(); - fleetServerContainerId = data?.fleetServerContainerId; }); on('after:run', async () => { const { log } = await stackServicesPromise; - if (fleetServerContainerId) { + if (fleetSrv) { try { - execa.sync('docker', ['kill', fleetServerContainerId]); + await fleetSrv.stop(); } catch (error) { log.error(error); } diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index 288d422663a07..dd6d758473baf 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -58,6 +58,7 @@ export const createAndEnrollEndpointHost = async ({ const { url: agentUrl } = await getAgentDownloadUrl(version, useClosestVersionMatch, log); const agentDownload = isRunningInCI ? await downloadAndStoreAgent(agentUrl) : undefined; + // TODO: remove dependency on env. var and keep function pure const hostVm = process.env.CI ? await createVm({ type: 'vagrant', @@ -102,7 +103,7 @@ export const createAndEnrollEndpointHost = async ({ */ export const destroyEndpointHost = async ( kbnClient: KbnClient, - createdHost: CreateAndEnrollEndpointHostResponse + createdHost: Pick ): Promise => { await Promise.all([ deleteMultipassVm(createdHost.hostname), diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 38450f6afff56..87341bd156eb9 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -94,7 +94,7 @@ interface StartFleetServerOptions { port?: number; } -interface StartedFleetServer extends StartedServer { +export interface StartedFleetServer extends StartedServer { /** The Fleet Agent policy id that the fleet-server agent is running with */ policyId: string; } diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts index d4ad9a11290a4..e1e7ddc66595f 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts @@ -11,9 +11,14 @@ import { } from '../common/fleet_server/fleet_server_services'; import { getRuntimeServices } from './runtime'; +/** + * @deprecated + */ export const runFleetServerIfNeeded = async (): Promise< { fleetServerContainerId: string; fleetServerAgentPolicyId: string | undefined } | undefined > => { + // TODO:PT remove dependencies on this module and then delete this and just call the code below directly from the CLI tool + const { log, kbnClient } = getRuntimeServices(); // Runs under CI should force fleet server to be installed because the CI process first From c12b0eca2b044d995aa95181c71a9683916d019d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 26 Oct 2023 17:56:22 -0400 Subject: [PATCH 11/72] Fix bug in vagrant VM creation --- .../security_solution/scripts/endpoint/common/vm_services.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 8d7c51ba79024..580a3c51454e7 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -9,7 +9,7 @@ import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import chalk from 'chalk'; import { userInfo } from 'os'; -import { resolve, dirname } from 'path'; +import { join as pathJoin, dirname } from 'path'; import type { DownloadedAgentInfo } from './agent_downloads_service'; import { BaseDataGenerator } from '../../../common/endpoint/data_generators/base_data_generator'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; @@ -193,7 +193,7 @@ const createVagrantVm = async ({ name, log = createToolingLogger(), agentDownload: { fullFilePath: agentFullFilePath, filename: agentFileName }, - vagrantFile = resolve('../endpoint_agent_runner/Vagrantfile'), + vagrantFile = pathJoin(__dirname, 'vagrant', 'Vagrantfile'), memory, cpus, disk, From 95a3a0145eec6aa55ddf9eda8a9eb307f9836559 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 27 Oct 2023 10:53:30 -0400 Subject: [PATCH 12/72] Fix download/extract of agent package for Vagrant VMs --- .../scripts/endpoint/common/fleet_services.ts | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 605cf6919318d..94178c0964fe9 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -620,26 +620,30 @@ export const enrollHostVmWithFleet = async ({ log.info(`Installing Elastic Agent`); - // Mount the directory where the agent download cache is located - if (useAgentCache) { - const hostVmDownloadsDir = '/home/ubuntu/_agent_downloads'; + // For multipass, we need to place the Agent archive in the VM - either mounting local cache + // directory or downloading it directly from inside of the VM. + // For Vagrant, the archive is already in the VM - it was done during VM creation. + if (hostVm.type === 'multipass') { + if (useAgentCache) { + const hostVmDownloadsDir = '/home/ubuntu/_agent_downloads'; - log.debug( - `Mounting agents download cache directory [${agentDownload.directory}] to Host VM at [${hostVmDownloadsDir}]` - ); - const downloadsMount = await hostVm.mount(agentDownload.directory, hostVmDownloadsDir); + log.debug( + `Mounting agents download cache directory [${agentDownload.directory}] to Host VM at [${hostVmDownloadsDir}]` + ); + const downloadsMount = await hostVm.mount(agentDownload.directory, hostVmDownloadsDir); - log.debug(`Extracting download archive on host VM`); - await hostVm.exec(`tar -zxf ${downloadsMount.hostDir}/${agentDownload.filename}`); + log.debug(`Extracting download archive on host VM`); + await hostVm.exec(`tar -zxf ${downloadsMount.hostDir}/${agentDownload.filename}`); - await downloadsMount.unmount(); - } else { - log.debug(`Downloading Elastic Agent to host VM`); - await hostVm.exec(`curl -L ${agentDownload.url} -o ${agentDownload.filename}`); + await downloadsMount.unmount(); + } else { + log.debug(`Downloading Elastic Agent to host VM`); + await hostVm.exec(`curl -L ${agentDownload.url} -o ${agentDownload.filename}`); - log.debug(`Extracting download archive on host VM`); - await hostVm.exec(`tar -zxf ${agentDownload.filename}`); - await hostVm.exec(`rm -f ${agentDownload.filename}`); + log.debug(`Extracting download archive on host VM`); + await hostVm.exec(`tar -zxf ${agentDownload.filename}`); + await hostVm.exec(`rm -f ${agentDownload.filename}`); + } } const policyId = agentPolicyId || (await getOrCreateDefaultAgentPolicy({ kbnClient, log })).id; From 6d7a4e60eb5a7e9cf1f1d66af3888a2c5530a047 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 27 Oct 2023 11:02:29 -0400 Subject: [PATCH 13/72] add `getHostVmClient()` to endpoint host services --- .../endpoint/common/endpoint_host_services.ts | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index dd6d758473baf..a89183fd91f26 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -111,26 +111,18 @@ export const destroyEndpointHost = async ( ]); }; -export const deleteMultipassVm = async (vmName: string): Promise => { - const hostVm = process.env.CI - ? createVagrantHostVmClient(vmName) - : createMultipassHostVmClient(vmName); +const getHostVmClient = (vmName: string): HostVm => { + return process.env.CI ? createVagrantHostVmClient(vmName) : createMultipassHostVmClient(vmName); +}; - await hostVm.destroy(); +export const deleteMultipassVm = async (vmName: string): Promise => { + await getHostVmClient(vmName).destroy(); }; export async function stopEndpointHost(hostName: string): Promise { - const hostVm = process.env.CI - ? createVagrantHostVmClient(hostName) - : createMultipassHostVmClient(hostName); - - await hostVm.stop(); + await getHostVmClient(hostName).stop(); } export async function startEndpointHost(hostName: string): Promise { - const hostVm = process.env.CI - ? createVagrantHostVmClient(hostName) - : createMultipassHostVmClient(hostName); - - await hostVm.start(); + await getHostVmClient(hostName).start(); } From 0d1bfea2c79713a2f6be405591c24cb3ca7eebdd Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 27 Oct 2023 11:53:37 -0400 Subject: [PATCH 14/72] fix `exec()` for Vagrant vms --- .../security_solution/scripts/endpoint/common/vm_services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 580a3c51454e7..4cb266ea90213 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -254,7 +254,7 @@ export const createVagrantHostVmClient = ( log: ToolingLog = createToolingLogger() ): HostVm => { const exec = async (command: string): Promise => { - const execResponse = await execa.command(`vagrant ssh ${name} --command="${command}"`); + const execResponse = await execa.command(`vagrant ssh ${name} -- ${command}`); log.verbose(execResponse); From 6af9982bc83b3fc7cdb8aa658f8fc2f91193ec7d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 27 Oct 2023 17:56:30 -0400 Subject: [PATCH 15/72] fix vagrant commands to execute from the vagrantfile directory --- .../scripts/endpoint/common/vm_services.ts | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 4cb266ea90213..861a00de1862c 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -16,6 +16,7 @@ import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils import type { HostVm, HostVmExecResponse, SupportedVmManager } from './types'; const baseGenerator = new BaseDataGenerator(); +const DEFAULT_VAGRANTFILE = pathJoin(__dirname, 'vagrant', 'Vagrantfile'); export interface BaseVmCreateOptions { name: string; @@ -193,7 +194,7 @@ const createVagrantVm = async ({ name, log = createToolingLogger(), agentDownload: { fullFilePath: agentFullFilePath, filename: agentFileName }, - vagrantFile = pathJoin(__dirname, 'vagrant', 'Vagrantfile'), + vagrantFile = DEFAULT_VAGRANTFILE, memory, cpus, disk, @@ -248,13 +249,48 @@ const createVagrantVm = async ({ * Creates a generic interface (`HotVm`) for interacting with a VM creatd by Vagrant * @param name * @param log + * @param vagrantFile */ export const createVagrantHostVmClient = ( name: string, - log: ToolingLog = createToolingLogger() + log: ToolingLog = createToolingLogger(), + vagrantFile: string = DEFAULT_VAGRANTFILE ): HostVm => { + const VAGRANT_CWD = dirname(vagrantFile); + const execaOptions: execa.Options = { + env: { + VAGRANT_CWD, + }, + }; + + log.verbose(`Creating Vagrant VM client for [${name}] with vagrantfile [${vagrantFile}]`); + const exec = async (command: string): Promise => { - const execResponse = await execa.command(`vagrant ssh ${name} -- ${command}`); + // + // + // + // FIXME:PT Remove once output is captured + // + // + // + log.info(`${'-'.repeat(100)} +Machine name: ${name} +VAGRANT COMMAND: vagrant global-status --machine-readable +${'-'.repeat(100)} + +${(await execa.command('vagrant global-status --machine-readable', execaOptions)).stdout} + +${'-'.repeat(100)} +${'-'.repeat(100)} +VAGRANT COMMAND: vagrant status --machine-readable +${'-'.repeat(100)} + +${(await execa.command('vagrant status --machine-readable', execaOptions)).stdout} + +${'-'.repeat(100)} +`); + + const execResponse = await execa.command(`vagrant ssh ${name} -- ${command}`, execaOptions); log.verbose(execResponse); @@ -266,10 +302,7 @@ export const createVagrantHostVmClient = ( }; const destroy = async (): Promise => { - const destroyResponse = await execa.command(`vagrant destroy ${name} -f`, { - // Only `pipe` STDERR to parent process - stdio: ['inherit', 'inherit', 'pipe'], - }); + const destroyResponse = await execa.command(`vagrant destroy ${name} -f`, execaOptions); log.verbose(`VM [${name}] was destroyed successfully`, destroyResponse); }; @@ -292,11 +325,11 @@ export const createVagrantHostVmClient = ( }; const start = async () => { - await execa.command(`vagrant up ${name}`); + await execa.command(`vagrant up ${name}`, execaOptions); }; const stop = async () => { - await execa.command(`vagrant suspend ${name}`); + await execa.command(`vagrant suspend ${name}`, execaOptions); }; return { From 25624ffd4629b9e8a5e6a5cd7b85aef824bd5c1a Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 27 Oct 2023 19:47:44 -0400 Subject: [PATCH 16/72] more output of vagrant comments for debug --- .../scripts/endpoint/common/vm_services.ts | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 861a00de1862c..31daea27eceda 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -275,19 +275,41 @@ export const createVagrantHostVmClient = ( // log.info(`${'-'.repeat(100)} Machine name: ${name} +${'-'.repeat(100)} +VAGRANT COMMAND: vagrant global-status +${'-'.repeat(100)} + +${(await execa.command('vagrant global-status', execaOptions)).stdout} + +${'='.repeat(100)} +${'-'.repeat(100)} VAGRANT COMMAND: vagrant global-status --machine-readable ${'-'.repeat(100)} ${(await execa.command('vagrant global-status --machine-readable', execaOptions)).stdout} -${'-'.repeat(100)} +${'='.repeat(100)} ${'-'.repeat(100)} VAGRANT COMMAND: vagrant status --machine-readable ${'-'.repeat(100)} ${(await execa.command('vagrant status --machine-readable', execaOptions)).stdout} +${'='.repeat(100)} +${'-'.repeat(100)} +VAGRANT COMMAND: vagrant box list ${'-'.repeat(100)} + +${(await execa.command('vagrant box list', execaOptions)).stdout} + +${'='.repeat(100)} +${'-'.repeat(100)} +VAGRANT COMMAND: vagrant box list --machine-readable +${'-'.repeat(100)} + +${(await execa.command('vagrant box list --machine-readable', execaOptions)).stdout} + +${'='.repeat(100)} `); const execResponse = await execa.command(`vagrant ssh ${name} -- ${command}`, execaOptions); @@ -302,7 +324,7 @@ ${'-'.repeat(100)} }; const destroy = async (): Promise => { - const destroyResponse = await execa.command(`vagrant destroy ${name} -f`, execaOptions); + const destroyResponse = await execa.command(`vagrant destroy -f`, execaOptions); log.verbose(`VM [${name}] was destroyed successfully`, destroyResponse); }; @@ -325,11 +347,11 @@ ${'-'.repeat(100)} }; const start = async () => { - await execa.command(`vagrant up ${name}`, execaOptions); + await execa.command(`vagrant up`, execaOptions); }; const stop = async () => { - await execa.command(`vagrant suspend ${name}`, execaOptions); + await execa.command(`vagrant suspend`, execaOptions); }; return { From 9ffef2d2b41a0b512508e922043a748962198d99 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 27 Oct 2023 20:01:52 -0400 Subject: [PATCH 17/72] add host name to the virutalbox provider --- .../scripts/endpoint/common/vagrant/Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile index 92c4cba8d2dd4..77c04b2ca8e1c 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile @@ -13,6 +13,7 @@ Vagrant.configure("2") do |config| config.vm.provider :virtualbox do |vb| vb.memory = 4096 vb.cpus = 2 + vb.name = hostname end config.vm.provider :vmware_desktop do |v, override| From 90ca9f25a468d80eccde4c9bd899a1dd4973fbf0 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 30 Oct 2023 08:57:57 -0400 Subject: [PATCH 18/72] revert change to `Vagrantfile` --- .../scripts/endpoint/common/vagrant/Vagrantfile | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile index 77c04b2ca8e1c..92c4cba8d2dd4 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile @@ -13,7 +13,6 @@ Vagrant.configure("2") do |config| config.vm.provider :virtualbox do |vb| vb.memory = 4096 vb.cpus = 2 - vb.name = hostname end config.vm.provider :vmware_desktop do |v, override| From 4dfc4ae64cac35ab6a607624129d46d014bc9312 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 30 Oct 2023 09:04:43 -0400 Subject: [PATCH 19/72] adjust createVagrantHOstVmClient() --- .../scripts/endpoint/common/vm_services.ts | 56 ++----------------- 1 file changed, 5 insertions(+), 51 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 31daea27eceda..fb346b6a9046c 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -242,19 +242,19 @@ const createVagrantVm = async ({ throw e; } - return createVagrantHostVmClient(name, log); + return createVagrantHostVmClient(name, undefined, log); }; /** - * Creates a generic interface (`HotVm`) for interacting with a VM creatd by Vagrant + * Creates a generic interface (`HotVm`) for interacting with a VM created by Vagrant * @param name * @param log * @param vagrantFile */ export const createVagrantHostVmClient = ( name: string, - log: ToolingLog = createToolingLogger(), - vagrantFile: string = DEFAULT_VAGRANTFILE + vagrantFile: string = DEFAULT_VAGRANTFILE, + log: ToolingLog = createToolingLogger() ): HostVm => { const VAGRANT_CWD = dirname(vagrantFile); const execaOptions: execa.Options = { @@ -266,53 +266,7 @@ export const createVagrantHostVmClient = ( log.verbose(`Creating Vagrant VM client for [${name}] with vagrantfile [${vagrantFile}]`); const exec = async (command: string): Promise => { - // - // - // - // FIXME:PT Remove once output is captured - // - // - // - log.info(`${'-'.repeat(100)} -Machine name: ${name} -${'-'.repeat(100)} -VAGRANT COMMAND: vagrant global-status -${'-'.repeat(100)} - -${(await execa.command('vagrant global-status', execaOptions)).stdout} - -${'='.repeat(100)} -${'-'.repeat(100)} -VAGRANT COMMAND: vagrant global-status --machine-readable -${'-'.repeat(100)} - -${(await execa.command('vagrant global-status --machine-readable', execaOptions)).stdout} - -${'='.repeat(100)} -${'-'.repeat(100)} -VAGRANT COMMAND: vagrant status --machine-readable -${'-'.repeat(100)} - -${(await execa.command('vagrant status --machine-readable', execaOptions)).stdout} - -${'='.repeat(100)} -${'-'.repeat(100)} -VAGRANT COMMAND: vagrant box list -${'-'.repeat(100)} - -${(await execa.command('vagrant box list', execaOptions)).stdout} - -${'='.repeat(100)} -${'-'.repeat(100)} -VAGRANT COMMAND: vagrant box list --machine-readable -${'-'.repeat(100)} - -${(await execa.command('vagrant box list --machine-readable', execaOptions)).stdout} - -${'='.repeat(100)} -`); - - const execResponse = await execa.command(`vagrant ssh ${name} -- ${command}`, execaOptions); + const execResponse = await execa.command(`vagrant ssh -- ${command}`, execaOptions); log.verbose(execResponse); From be1140218e00f833200d85f71d14f01edbe6047a Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 1 Nov 2023 15:26:05 -0400 Subject: [PATCH 20/72] changes mainly for vagrant --- .../scripts/endpoint/common/vm_services.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index fb346b6a9046c..928ca02432648 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -102,10 +102,12 @@ export const createMultipassHostVmClient = ( const unmount = async (hostVmDir: string) => { await execa.command(`multipass unmount ${name}:${hostVmDir}`); + log.verbose(`multipass unmount response:\n`, response); }; const mount = async (localDir: string, hostVmDir: string) => { - await execa.command(`multipass mount ${localDir} ${name}:${hostVmDir}`); + const response = await execa.command(`multipass mount ${localDir} ${name}:${hostVmDir}`); + log.verbose(`multipass mount response:\n`, response); return { hostDir: hostVmDir, @@ -114,11 +116,13 @@ export const createMultipassHostVmClient = ( }; const start = async () => { - await execa.command(`multipass start ${name}`); + const response = await execa.command(`multipass start ${name}`); + log.verbose(`multipass start response:\n`, response); }; const stop = async () => { - await execa.command(`multipass stop ${name}`); + const response = await execa.command(`multipass stop ${name}`); + log.verbose(`multipass stop response:\n`, response); }; return { @@ -261,9 +265,10 @@ export const createVagrantHostVmClient = ( env: { VAGRANT_CWD, }, + stdio: ['inherit', 'pipe', 'pipe'], }; - log.verbose(`Creating Vagrant VM client for [${name}] with vagrantfile [${vagrantFile}]`); + log.debug(`Creating Vagrant VM client for [${name}] with vagrantfile [${vagrantFile}]`); const exec = async (command: string): Promise => { const execResponse = await execa.command(`vagrant ssh -- ${command}`, execaOptions); @@ -280,7 +285,7 @@ export const createVagrantHostVmClient = ( const destroy = async (): Promise => { const destroyResponse = await execa.command(`vagrant destroy -f`, execaOptions); - log.verbose(`VM [${name}] was destroyed successfully`, destroyResponse); + log.debug(`VM [${name}] was destroyed successfully`, destroyResponse); }; const info = () => { @@ -301,11 +306,13 @@ export const createVagrantHostVmClient = ( }; const start = async () => { - await execa.command(`vagrant up`, execaOptions); + const response = await execa.command(`vagrant up`, execaOptions); + log.verbose('vagrant up response:\n', response); }; const stop = async () => { - await execa.command(`vagrant suspend`, execaOptions); + const response = await execa.command(`vagrant suspend`, execaOptions); + log.verbose('vagrant suspend response:\n', response); }; return { From d9cabe836cc16b587f9ffa9a4b6fe9d7a8860b4c Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 1 Nov 2023 15:50:34 -0400 Subject: [PATCH 21/72] fix missingg var --- .../security_solution/scripts/endpoint/common/vm_services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 928ca02432648..5de708641eb07 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -101,7 +101,7 @@ export const createMultipassHostVmClient = ( }; const unmount = async (hostVmDir: string) => { - await execa.command(`multipass unmount ${name}:${hostVmDir}`); + const response = await execa.command(`multipass unmount ${name}:${hostVmDir}`); log.verbose(`multipass unmount response:\n`, response); }; From e2c0b11ea69c88e62c444e9a4eaacd91cd3b406d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 1 Nov 2023 17:38:25 -0400 Subject: [PATCH 22/72] still trying vagrant command to be successful --- .../security_solution/scripts/endpoint/common/fleet_services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 94178c0964fe9..c060bcea1eed4 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -655,7 +655,7 @@ export const enrollHostVmWithFleet = async ({ const agentEnrollCommand = [ 'sudo', - `/home/ubuntu/${agentUrlInfo.dirName}/elastic-agent`, + `./${agentUrlInfo.dirName}/elastic-agent`, 'install', From 4df5c387e615cb10ed3c2e7b739bad9a39b1efac Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 1 Nov 2023 17:07:34 -0400 Subject: [PATCH 23/72] Fix error when checking fleet-server standalone is registered with ES --- .../endpoint/common/fleet_server/fleet_server_services.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 87341bd156eb9..6a3d5244b1d5d 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -289,7 +289,7 @@ const startFleetServerWithDocker = async ({ await updateFleetElasticsearchOutputHostNames(kbnClient, log); if (isServerless) { - log.info(`Waiting for server to register with Elasticsearch`); + log.info(`Waiting for server [${hostname}] to register with Elasticsearch`); await waitForFleetServerToRegisterWithElasticsearch(kbnClient, hostname, 120000); } else { @@ -683,7 +683,7 @@ const waitForFleetServerToRegisterWithElasticsearch = async ( .then((response) => response.data) .catch(catchAxiosErrorFormatAndThrow); - return (fleetServerRecord.hits.total as estypes.SearchTotalHits).value === 1; + return ((fleetServerRecord?.hits?.total as estypes.SearchTotalHits)?.value ?? 0) === 1; }, RETRYABLE_TRANSIENT_ERRORS); if (!found) { From 722e473e97fa85cf010ef2004da293f5a141b2a1 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 09:57:58 -0400 Subject: [PATCH 24/72] change cypress setup to ensure fleet server is setup and running --- .../management/cypress/cypress_base.config.ts | 6 ++- .../management/cypress/support/common.ts | 33 ++++++++++++ .../cypress/support/data_loaders.ts | 51 ++----------------- .../cypress/support/fleet_server_setup.ts | 27 ++++++++++ .../cypress/support/response_actions.ts | 11 +--- .../fleet_server/fleet_server_services.ts | 24 +++++++++ 6 files changed, 94 insertions(+), 58 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/management/cypress/support/common.ts create mode 100644 x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts index 2e6023c7690a0..66cbcbb42150c 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts @@ -6,6 +6,7 @@ */ import { merge } from 'lodash'; +import { setupFleetServerForCypressTestRun } from './support/fleet_server_setup'; import { setupToolingLogLevel } from './support/setup_tooling_log_level'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import { dataLoaders, dataLoadersForRealEndpoints } from './support/data_loaders'; @@ -69,11 +70,14 @@ export const getCypressBaseConfig = ( experimentalRunAllSpecs: true, experimentalMemoryManagement: true, experimentalInteractiveRunEvents: true, - setupNodeEvents: (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => { + setupNodeEvents: async (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => { // IMPORTANT: setting the log level should happen before any tooling is called setupToolingLogLevel(config); + await setupFleetServerForCypressTestRun(on, config); + dataLoaders(on, config); + // Data loaders specific to "real" Endpoint testing dataLoadersForRealEndpoints(on, config); diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/common.ts b/x-pack/plugins/security_solution/public/management/cypress/support/common.ts new file mode 100644 index 0000000000000..6b4c30b884f8b --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/support/common.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; +import { createRuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; + +const RUNTIME_SERVICES_CACHE = new WeakMap(); + +export const setupStackServicesUsingCypressConfig = async (config: Cypress.PluginConfigOptions) => { + if (RUNTIME_SERVICES_CACHE.has(config)) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return RUNTIME_SERVICES_CACHE.get(config)!; + } + + const stackServices = await createRuntimeServices({ + kibanaUrl: config.env.KIBANA_URL, + elasticsearchUrl: config.env.ELASTICSEARCH_URL, + fleetServerUrl: config.env.FLEET_SERVER_URL, + username: config.env.KIBANA_USERNAME, + password: config.env.KIBANA_PASSWORD, + esUsername: config.env.ELASTICSEARCH_USERNAME, + esPassword: config.env.ELASTICSEARCH_PASSWORD, + asSuperuser: true, + }); + + RUNTIME_SERVICES_CACHE.set(config, stackServices); + + return stackServices; +}; diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 9808dac925966..85e2a2fe48c91 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -11,8 +11,7 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common'; import execa from 'execa'; import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; -import type { StartedFleetServer } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; -import { startFleetServer } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; +import { setupStackServicesUsingCypressConfig } from './common'; import type { KibanaKnownUserAccounts } from '../common/constants'; import { KIBANA_KNOWN_DEFAULT_ACCOUNTS } from '../common/constants'; import type { EndpointSecurityRoleNames } from '../../../../scripts/endpoint/common/roles_users'; @@ -60,7 +59,6 @@ import type { IndexedHostsAndAlertsResponse } from '../../../../common/endpoint/ import { deleteIndexedHostsAndAlerts } from '../../../../common/endpoint/index_data'; import type { IndexedCase } from '../../../../common/endpoint/data_loaders/index_case'; import { deleteIndexedCase, indexCase } from '../../../../common/endpoint/data_loaders/index_case'; -import { createRuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; import { deleteIndexedFleetEndpointPolicies, @@ -128,18 +126,7 @@ export const dataLoaders = ( ): void => { // Env. variable is set by `cypress_serverless.config.ts` const isServerless = config.env.IS_SERVERLESS; - - const stackServicesPromise = createRuntimeServices({ - kibanaUrl: config.env.KIBANA_URL, - elasticsearchUrl: config.env.ELASTICSEARCH_URL, - fleetServerUrl: config.env.FLEET_SERVER_URL, - username: config.env.KIBANA_USERNAME, - password: config.env.KIBANA_PASSWORD, - esUsername: config.env.ELASTICSEARCH_USERNAME, - esPassword: config.env.ELASTICSEARCH_PASSWORD, - asSuperuser: true, - }); - + const stackServicesPromise = setupStackServicesUsingCypressConfig(config); const roleAndUserLoaderPromise: Promise = stackServicesPromise.then( ({ kbnClient, log }) => { return new TestRoleAndUserLoader(kbnClient, log, isServerless); @@ -290,39 +277,7 @@ export const dataLoadersForRealEndpoints = ( on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions ): void => { - let fleetSrv: StartedFleetServer | undefined; - - const stackServicesPromise = createRuntimeServices({ - kibanaUrl: config.env.KIBANA_URL, - elasticsearchUrl: config.env.ELASTICSEARCH_URL, - fleetServerUrl: config.env.FLEET_SERVER_URL, - username: config.env.KIBANA_USERNAME, - password: config.env.KIBANA_PASSWORD, - esUsername: config.env.ELASTICSEARCH_USERNAME, - esPassword: config.env.ELASTICSEARCH_PASSWORD, - asSuperuser: true, - }); - - on('before:run', async () => { - const { kbnClient, log } = await stackServicesPromise; - - fleetSrv = await startFleetServer({ - kbnClient, - logger: log, - force: true, - }); - }); - - on('after:run', async () => { - const { log } = await stackServicesPromise; - if (fleetSrv) { - try { - await fleetSrv.stop(); - } catch (error) { - log.error(error); - } - } - }); + const stackServicesPromise = setupStackServicesUsingCypressConfig(config); on('task', { createEndpointHost: async ( diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts b/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts new file mode 100644 index 0000000000000..394469fce31f7 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { startFleetServerIfNecessary } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; +import { setupStackServicesUsingCypressConfig } from './common'; + +export const setupFleetServerForCypressTestRun = async ( + on: Cypress.PluginEvents, + config: Cypress.PluginConfigOptions +) => { + const { kbnClient, log } = await setupStackServicesUsingCypressConfig(config); + + const startedFleetServer = await startFleetServerIfNecessary({ + kbnClient, + logger: log, + }).catch(log.error); + + on('after:run', async () => { + if (startedFleetServer) { + await startedFleetServer.stop(); + } + }); +}; diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts b/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts index b22e50c660d83..8d88e4d655d9d 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts @@ -9,25 +9,18 @@ import { get } from 'lodash'; +import { setupStackServicesUsingCypressConfig } from './common'; import { getLatestActionDoc, updateActionDoc, waitForNewActionDoc, } from '../../../../scripts/endpoint/common/response_actions'; -import { createRuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; export const responseActionTasks = ( on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions ): void => { - const stackServicesPromise = createRuntimeServices({ - kibanaUrl: config.env.KIBANA_URL, - elasticsearchUrl: config.env.ELASTICSEARCH_URL, - fleetServerUrl: config.env.FLEET_SERVER_URL, - username: config.env.KIBANA_USERNAME, - password: config.env.KIBANA_PASSWORD, - asSuperuser: true, - }); + const stackServicesPromise = setupStackServicesUsingCypressConfig(config); on('task', { getLatestActionDoc: async () => { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 6a3d5244b1d5d..30cc380c17889 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -99,6 +99,15 @@ export interface StartedFleetServer extends StartedServer { policyId: string; } +/** + * Starts Fleet Server and connectors it to the stack + * @param kbnClient + * @param logger + * @param policy + * @param version + * @param force + * @param port + */ export const startFleetServer = async ({ kbnClient, logger, @@ -137,6 +146,18 @@ export const startFleetServer = async ({ }); }; +/** + * Checks if fleet server is already running and if not, then it will attempt to start + * one and connect it to the stack + */ +export const startFleetServerIfNecessary = async ( + options: StartFleetServerOptions +): Promise => { + if (options.force || !(await isFleetServerRunning(options.kbnClient, options.logger))) { + return startFleetServer(options); + } +}; + const getOrCreateFleetServerAgentPolicyId = async ( kbnClient: KbnClient, log: ToolingLog @@ -334,6 +355,9 @@ Kill container: ${chalk.cyan(`docker kill ${containerId}`)} url: fleetServerUrl, info, stop: async () => { + log.info( + `Stopping (kill) fleet server. Container name [${containerName}] id [${containerId}]` + ); await execa('docker', ['kill', containerId]); }, }; From 96e5e1ea6951ffd166e5a4a0e06b4d3aeb522cdb Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 13:08:22 -0400 Subject: [PATCH 25/72] Add setup of fleet to cypress parallel runner based on env. variable --- .../management/cypress/cypress_base.config.ts | 8 +- .../cypress/support/fleet_server_setup.ts | 2 + .../scripts/run_cypress/parallel.ts | 97 ++++++++++++------- 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts index 66cbcbb42150c..caaa542329028 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts @@ -6,7 +6,6 @@ */ import { merge } from 'lodash'; -import { setupFleetServerForCypressTestRun } from './support/fleet_server_setup'; import { setupToolingLogLevel } from './support/setup_tooling_log_level'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import { dataLoaders, dataLoadersForRealEndpoints } from './support/data_loaders'; @@ -57,6 +56,11 @@ export const getCypressBaseConfig = ( // to `debug` or `verbose` when wanting to debug tooling used by tests (ex. data indexer functions). TOOLING_LOG_LEVEL: 'info', + // Variable works in conjunction with the Cypress parallel runner. When set to true, fleet server + // will be setup right after the Kibana stack, so that by the time cypress tests `.run()`/`.open()`, + // the env. will be all setup and we don't have to explicitly setup fleet from a test file + WITH_FLEET_SERVER: true, + // grep related configs grepFilterSpecs: true, grepOmitFiltered: true, @@ -74,8 +78,6 @@ export const getCypressBaseConfig = ( // IMPORTANT: setting the log level should happen before any tooling is called setupToolingLogLevel(config); - await setupFleetServerForCypressTestRun(on, config); - dataLoaders(on, config); // Data loaders specific to "real" Endpoint testing diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts b/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts index 394469fce31f7..802c7b28061cf 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts @@ -8,6 +8,8 @@ import { startFleetServerIfNecessary } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; import { setupStackServicesUsingCypressConfig } from './common'; +// FIXME:PT delete this + export const setupFleetServerForCypressTestRun = async ( on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 19eeafaecfcdc..2464f547a9f67 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -10,7 +10,6 @@ import yargs from 'yargs'; import _ from 'lodash'; import globby from 'globby'; import pMap from 'p-map'; -import { ToolingLog } from '@kbn/tooling-log'; import { withProcRunner } from '@kbn/dev-proc-runner'; import cypress from 'cypress'; import { findChangedFiles } from 'find-cypress-specs'; @@ -27,6 +26,10 @@ import { import { createFailError } from '@kbn/dev-cli-errors'; import pRetry from 'p-retry'; +import { createToolingLogger } from '../../common/endpoint/data_loaders/utils'; +import { createKbnClient } from '../endpoint/common/stack_services'; +import type { StartedFleetServer } from '../endpoint/common/fleet_server/fleet_server_services'; +import { startFleetServerIfNecessary } from '../endpoint/common/fleet_server/fleet_server_services'; import { renderSummaryTable } from './print_run'; import { isSkipped, parseTestFileConfig } from './utils'; import { getFTRConfig } from './get_ftr_config'; @@ -62,12 +65,7 @@ const retrieveIntegrations = (integrationsPaths: string[]) => { export const cli = () => { run( - async () => { - const log = new ToolingLog({ - level: 'info', - writeTo: process.stdout, - }); - + async ({ log: _cliLogger }) => { const { argv } = yargs(process.argv.slice(2)) .coerce('configFile', (arg) => (_.isArray(arg) ? _.last(arg) : arg)) .coerce('spec', (arg) => (_.isArray(arg) ? _.last(arg) : arg)) @@ -84,7 +82,7 @@ export const cli = () => { ) .boolean('inspect'); - log.info(` + _cliLogger.info(` ---------------------------------------------- Script arguments: ---------------------------------------------- @@ -95,10 +93,15 @@ ${JSON.stringify(argv, null, 2)} `); const isOpen = argv._.includes('open'); - const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); + if (cypressConfigFile.env?.TOOLING_LOG_LEVEL) { + createToolingLogger.defaultLogLevel = cypressConfigFile.env.TOOLING_LOG_LEVEL; + } + + const log = createToolingLogger(); + log.info(` ---------------------------------------------- Cypress config for file: ${cypressConfigFilePath}: @@ -263,6 +266,34 @@ ${JSON.stringify(cypressConfigFile, null, 2)} isOpen, }); + const createUrlFromFtrConfig = ( + type: 'elasticsearch' | 'kibana' | 'fleetserver', + withAuth: boolean = false + ): string => { + const getKeyPath = (keyPath: string = ''): string => { + return `servers.${type}${keyPath ? `.${keyPath}` : ''}`; + }; + + if (!config.get(getKeyPath())) { + throw new Error(`Unable to create URL for ${type}. Not found in FTR config at `); + } + + const url = new URL('http://localhost'); + + url.port = config.get(getKeyPath('port')); + url.protocol = config.get(getKeyPath('protocol')); + url.hostname = config.get(getKeyPath('hostname')); + + if (withAuth) { + url.username = config.get(getKeyPath('username')); + url.password = config.get(getKeyPath('password')); + } + + return url.toString().replace(/\/$/, ''); + }; + + const baseUrl = createUrlFromFtrConfig('kibana'); + log.info(` ---------------------------------------------- Cypress FTR setup for file: ${filePath}: @@ -323,6 +354,22 @@ ${JSON.stringify( inspect: argv.inspect, }); + // Setup fleet if Cypress config requires it + let fleetServer: void | StartedFleetServer; + if (cypressConfigFile.env?.WITH_FLEET_SERVER) { + const kbnClient = createKbnClient({ + url: baseUrl, + username: config.get('servers.kibana.username'), + password: config.get('servers.kibana.password'), + log, + }); + + fleetServer = await startFleetServerIfNecessary({ + kbnClient, + logger: log, + }); + } + await providers.loadAll(); const functionalTestRunner = new FunctionalTestRunner( @@ -331,34 +378,6 @@ ${JSON.stringify( EsVersion.getDefault() ); - const createUrlFromFtrConfig = ( - type: 'elasticsearch' | 'kibana' | 'fleetserver', - withAuth: boolean = false - ): string => { - const getKeyPath = (keyPath: string = ''): string => { - return `servers.${type}${keyPath ? `.${keyPath}` : ''}`; - }; - - if (!config.get(getKeyPath())) { - throw new Error(`Unable to create URL for ${type}. Not found in FTR config at `); - } - - const url = new URL('http://localhost'); - - url.port = config.get(getKeyPath('port')); - url.protocol = config.get(getKeyPath('protocol')); - url.hostname = config.get(getKeyPath('hostname')); - - if (withAuth) { - url.username = config.get(getKeyPath('username')); - url.password = config.get(getKeyPath('password')); - } - - return url.toString().replace(/\/$/, ''); - }; - - const baseUrl = createUrlFromFtrConfig('kibana'); - const ftrEnv = await pRetry(() => functionalTestRunner.run(abortCtrl.signal), { retries: 1, }); @@ -437,6 +456,10 @@ ${JSON.stringify(cyCustomEnv, null, 2)} } } + if (fleetServer) { + await fleetServer.stop(); + } + await procs.stop('kibana'); await shutdownEs(); cleanupServerPorts({ esPort, kibanaPort, fleetServerPort }); From aa1d3bdb3a4da541aa6d5cfe7fa66c04d512091d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 14:05:04 -0400 Subject: [PATCH 26/72] add log entry to start of fleet server --- .../plugins/security_solution/scripts/run_cypress/parallel.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 2464f547a9f67..8d766059bbb72 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -357,6 +357,8 @@ ${JSON.stringify( // Setup fleet if Cypress config requires it let fleetServer: void | StartedFleetServer; if (cypressConfigFile.env?.WITH_FLEET_SERVER) { + log.info(`Starting fleet-server`); + const kbnClient = createKbnClient({ url: baseUrl, username: config.get('servers.kibana.username'), From d39a3a9b60aec4cf398caa046ffede8f06734002 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 14:15:10 -0400 Subject: [PATCH 27/72] set cypress logging level to debug --- .../public/management/cypress/cypress_base.config.ts | 2 +- .../endpoint/common/fleet_server/fleet_server_services.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts index caaa542329028..555e02085537c 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts @@ -54,7 +54,7 @@ export const getCypressBaseConfig = ( // Default log level for instance of `ToolingLog` created via `crateToolingLog()`. Set this // to `debug` or `verbose` when wanting to debug tooling used by tests (ex. data indexer functions). - TOOLING_LOG_LEVEL: 'info', + TOOLING_LOG_LEVEL: 'debug', // Variable works in conjunction with the Cypress parallel runner. When set to true, fleet server // will be setup right after the Kibana stack, so that by the time cypress tests `.run()`/`.open()`, diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 30cc380c17889..8eb3bb9805677 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -653,12 +653,12 @@ export const isFleetServerRunning = async ( httpsAgent: new https.Agent({ rejectUnauthorized: false }), }) .then((response) => { - log.verbose(`Fleet server is up and running as [${fleetServerUrl}]`, response.data); + log.debug(`Fleet server is up and running as [${fleetServerUrl}]`, response.data); return true; }) .catch(catchAxiosErrorFormatAndThrow) .catch((e) => { - log.verbose(`Fleet server not up. Attempt to call [${url.toString()}] failed with:`, e); + log.debug(`Fleet server not up. Attempt to call [${url.toString()}] failed with:`, e); return false; }); }; From 277d4cc190f0ea48457d411125ba4d28fcbf0acc Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 15:44:30 -0400 Subject: [PATCH 28/72] add log statements --- .../management/cypress/support/setup_tooling_log_level.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts b/x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts index c4c1acd428355..b4901bef9321a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts @@ -13,12 +13,13 @@ import { createToolingLogger } from '../../../../common/endpoint/data_loaders/ut * @param config */ export const setupToolingLogLevel = (config: Cypress.PluginConfigOptions) => { + const log = createToolingLogger(); const defaultToolingLogLevel = config.env.TOOLING_LOG_LEVEL; + log.info(`Cypress config 'env.TOOLING_LOG_LEVEL': ${defaultToolingLogLevel}`); + if (defaultToolingLogLevel && defaultToolingLogLevel !== createToolingLogger.defaultLogLevel) { createToolingLogger.defaultLogLevel = defaultToolingLogLevel; - createToolingLogger().info( - `Default log level for 'createToolingLogger()' set to ${defaultToolingLogLevel}` - ); + log.info(`Default log level for 'createToolingLogger()' set to ${defaultToolingLogLevel}`); } }; From 8676d1b44c7eb48f291cd5f65cf04f784be7abca Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 15:55:36 -0400 Subject: [PATCH 29/72] ensure fleet server is started on the port defined in the FTR config --- .../plugins/security_solution/scripts/run_cypress/parallel.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 8d766059bbb72..87062283a937f 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -369,6 +369,9 @@ ${JSON.stringify( fleetServer = await startFleetServerIfNecessary({ kbnClient, logger: log, + port: config.has('servers.fleetserver.port') + ? (config.get('servers.fleetserver.port') as number) + : undefined, }); } From 41a2dfcb765eabb610dcc84b68e15407aa259890 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 16:50:47 -0400 Subject: [PATCH 30/72] more debug messages --- .../scripts/endpoint/common/fleet_services.ts | 1 + .../plugins/security_solution/scripts/run_cypress/parallel.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index c060bcea1eed4..3146f45babe39 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -559,6 +559,7 @@ export const getFleetElasticsearchOutputHost = async (kbnClient: KbnClient): Pro } if (!host) { + log.error(`Outputs returned from Fleet:`, outputs); throw new Error(`An output for Elasticsearch was not found in Fleet settings`); } diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 87062283a937f..964ba40913033 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -357,7 +357,7 @@ ${JSON.stringify( // Setup fleet if Cypress config requires it let fleetServer: void | StartedFleetServer; if (cypressConfigFile.env?.WITH_FLEET_SERVER) { - log.info(`Starting fleet-server`); + log.info(`Setting fleet-server for this Cypress config`); const kbnClient = createKbnClient({ url: baseUrl, @@ -366,6 +366,8 @@ ${JSON.stringify( log, }); + log.info(`is Kibana up?\n`, JSON.stringify(await kbnClient.status.get(), null, 2)); + fleetServer = await startFleetServerIfNecessary({ kbnClient, logger: log, From 5bf5de9d6e78e4a6d5edc78da40c903367f064ae Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 17:25:15 -0400 Subject: [PATCH 31/72] fix log var not being defined --- .../scripts/endpoint/common/fleet_services.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 3146f45babe39..6959da006a8f5 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -548,7 +548,10 @@ export const fetchFleetOutputs = async (kbnClient: KbnClient): Promise => { +export const getFleetElasticsearchOutputHost = async ( + kbnClient: KbnClient, + log: ToolingLog = createToolingLogger() +): Promise => { const outputs = await fetchFleetOutputs(kbnClient); let host: string = ''; @@ -559,7 +562,7 @@ export const getFleetElasticsearchOutputHost = async (kbnClient: KbnClient): Pro } if (!host) { - log.error(`Outputs returned from Fleet:`, outputs); + log.error(`Outputs returned from Fleet:\n${JSON.stringify(outputs, null, 2)}`); throw new Error(`An output for Elasticsearch was not found in Fleet settings`); } From af18f093fb9c8386791edc5436962fbcf63fd4a5 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 2 Nov 2023 17:40:43 -0400 Subject: [PATCH 32/72] ensure fleet setup api is called prior to creating fleet server --- .../fleet_server/fleet_server_services.ts | 3 ++ .../scripts/endpoint/common/fleet_services.ts | 28 ++++++++++++++++++- .../scripts/run_cypress/parallel.ts | 2 -- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 8eb3bb9805677..ed1d478fe69fb 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -51,6 +51,7 @@ import { isServerlessKibanaFlavor } from '../stack_services'; import type { FormattedAxiosError } from '../format_axios_error'; import { catchAxiosErrorFormatAndThrow } from '../format_axios_error'; import { + ensureFleetSetup, fetchFleetOutputs, fetchFleetServerHostList, fetchFleetServerUrl, @@ -126,6 +127,8 @@ export const startFleetServer = async ({ ); } + await ensureFleetSetup(kbnClient, logger); + const isServerless = await isServerlessKibanaFlavor(kbnClient); const policyId = policy || !isServerless ? await getOrCreateFleetServerAgentPolicyId(kbnClient, logger) : ''; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 6959da006a8f5..1f6efee2906b2 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { map, pick } from 'lodash'; +import { map, memoize, pick } from 'lodash'; import type { Client, estypes } from '@elastic/elasticsearch'; import type { Agent, @@ -23,6 +23,7 @@ import type { PackagePolicy, GetInfoResponse, GetOneAgentPolicyResponse, + PostFleetSetupResponse, } from '@kbn/fleet-plugin/common'; import { AGENT_API_ROUTES, @@ -35,6 +36,7 @@ import { APP_API_ROUTES, epmRouteService, PACKAGE_POLICY_API_ROUTES, + SETUP_API_ROUTE, } from '@kbn/fleet-plugin/common'; import type { ToolingLog } from '@kbn/tooling-log'; import type { KbnClient } from '@kbn/test'; @@ -1111,3 +1113,27 @@ export const addEndpointIntegrationToAgentPolicy = async ({ return newIntegrationPolicy; }; + +/** + * Calls the fleet setup API to ensure fleet configured with default settings + * @param kbnClient + * @param log + */ +export const ensureFleetSetup = memoize( + async (kbnClient: KbnClient, log: ToolingLog): Promise => { + const setupResponse = await kbnClient + .request({ + path: SETUP_API_ROUTE, + headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 }, + method: 'POST', + }) + .catch(catchAxiosErrorFormatAndThrow); + + if (!setupResponse.data.isInitialized) { + log.verbose(`Fleet setup response:`, setupResponse); + throw new Error(`Call to initialize Fleet [${SETUP_API_ROUTE}] failed`); + } + + return setupResponse.data; + } +); diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 964ba40913033..3540529c62de6 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -366,8 +366,6 @@ ${JSON.stringify( log, }); - log.info(`is Kibana up?\n`, JSON.stringify(await kbnClient.status.get(), null, 2)); - fleetServer = await startFleetServerIfNecessary({ kbnClient, logger: log, From e1f475b20efc0d34cc7f04eb3b51c328809daab6 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 3 Nov 2023 10:27:08 -0400 Subject: [PATCH 33/72] fix cy task - ensure `null` is returned --- .../public/management/cypress/support/data_loaders.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 85e2a2fe48c91..39f17ea3462aa 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -423,11 +423,13 @@ export const dataLoadersForRealEndpoints = ( }, stopEndpointHost: async (hostName) => { - return stopEndpointHost(hostName); + await stopEndpointHost(hostName); + return null; }, startEndpointHost: async (hostName) => { - return startEndpointHost(hostName); + await startEndpointHost(hostName); + return null; }, }); }; From c4371b262935250c9db74f35ca724f1a2279cffc Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Sat, 4 Nov 2023 11:37:14 -0400 Subject: [PATCH 34/72] Remove usages that check for `CI` env. and use `getHostVmClient()` instead --- .../cypress/support/agent_actions.ts | 60 ++++--------------- .../cypress/support/data_loaders.ts | 50 ++-------------- .../endpoint/common/endpoint_host_services.ts | 8 +-- .../scripts/endpoint/common/types.ts | 8 +++ .../scripts/endpoint/common/vm_services.ts | 52 +++++++++++++++- 5 files changed, 77 insertions(+), 101 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/agent_actions.ts b/x-pack/plugins/security_solution/public/management/cypress/support/agent_actions.ts index 11a8b30a5e187..5ad564aaa14ba 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/agent_actions.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/agent_actions.ts @@ -6,10 +6,8 @@ */ // / -import type { ExecaReturnValue } from 'execa'; -import execa from 'execa'; -import { VAGRANT_CWD } from '../../../../scripts/endpoint/common/endpoint_host_services'; +import { getHostVmClient } from '../../../../scripts/endpoint/common/vm_services'; export const agentActions = (on: Cypress.PluginEvents): void => { on('task', { @@ -20,40 +18,19 @@ export const agentActions = (on: Cypress.PluginEvents): void => { hostname: string; uninstallToken?: string; }): Promise => { - let result; + const hostVmClient = getHostVmClient(hostname); + try { - if (process.env.CI) { - result = await execa( - 'vagrant', - [ - 'ssh', - '--', - `sudo elastic-agent uninstall -f ${ - uninstallToken ? `--uninstall-token ${uninstallToken}` : '' - }`, - ], - { - env: { - VAGRANT_CWD, - }, - } - ); - } else { - result = await execa(`multipass`, [ - 'exec', - hostname, - '--', - 'sh', - '-c', + return ( + await hostVmClient.exec( `sudo elastic-agent uninstall -f ${ uninstallToken ? `--uninstall-token ${uninstallToken}` : '' - }`, - ]); - } + }` + ) + ).stdout; } catch (err) { return err.stderr; } - return result.stdout; }, isAgentAndEndpointUninstalledFromHost: async ({ @@ -62,25 +39,10 @@ export const agentActions = (on: Cypress.PluginEvents): void => { hostname: string; uninstallToken?: string; }): Promise => { - let execaReturnValue: ExecaReturnValue; - if (process.env.CI) { - execaReturnValue = await execa('vagrant', ['ssh', '--', `ls /opt/Elastic`], { - env: { - VAGRANT_CWD, - }, - }); - } else { - execaReturnValue = await execa(`multipass`, [ - 'exec', - hostname, - '--', - 'sh', - '-c', - `ls /opt/Elastic`, - ]); - } + const hostVmClient = getHostVmClient(hostname); + const lsOutput = await hostVmClient.exec('ls /opt/Elastic'); - if (execaReturnValue.stdout === '') { + if (lsOutput.stdout === '') { return true; } diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 39f17ea3462aa..dc54c6e4db710 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -11,6 +11,7 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common'; import execa from 'execa'; import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; +import { getHostVmClient } from '../../../../scripts/endpoint/common/vm_services'; import { setupStackServicesUsingCypressConfig } from './common'; import type { KibanaKnownUserAccounts } from '../common/constants'; import { KIBANA_KNOWN_DEFAULT_ACCOUNTS } from '../common/constants'; @@ -34,7 +35,6 @@ import { destroyEndpointHost, startEndpointHost, stopEndpointHost, - VAGRANT_CWD, } from '../../../../scripts/endpoint/common/endpoint_host_services'; import type { IndexedEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_endpoint_policy_response'; import { @@ -335,15 +335,7 @@ export const dataLoadersForRealEndpoints = ( path: string; content: string; }): Promise => { - if (process.env.CI) { - await execa('vagrant', ['ssh', '--', `echo ${content} > ${path}`], { - env: { - VAGRANT_CWD, - }, - }); - } else { - await execa(`multipass`, ['exec', hostname, '--', 'sh', '-c', `echo ${content} > ${path}`]); - } + await getHostVmClient(hostname).exec(`echo ${content} > ${path}`); return null; }, @@ -356,16 +348,7 @@ export const dataLoadersForRealEndpoints = ( srcPath: string; destPath: string; }): Promise => { - if (process.env.CI) { - await execa('vagrant', ['upload', srcPath, destPath], { - env: { - VAGRANT_CWD, - }, - }); - } else { - await execa(`multipass`, ['transfer', srcPath, `${hostname}:${destPath}`]); - } - + await getHostVmClient(hostname).transfer(srcPath, destPath); return null; }, @@ -396,30 +379,9 @@ export const dataLoadersForRealEndpoints = ( path: string; password?: string; }): Promise => { - let result; - - if (process.env.CI) { - result = await execa( - `vagrant`, - ['ssh', '--', `unzip -p ${password ? `-P ${password} ` : ''}${path}`], - { - env: { - VAGRANT_CWD, - }, - } - ); - } else { - result = await execa(`multipass`, [ - 'exec', - hostname, - '--', - 'sh', - '-c', - `unzip -p ${password ? `-P ${password} ` : ''}${path}`, - ]); - } - - return result.stdout; + return ( + await getHostVmClient(hostname).exec(`unzip -p ${password ? `-P ${password} ` : ''}${path}`) + ).stdout; }, stopEndpointHost: async (hostName) => { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index a89183fd91f26..b48f5d1309a01 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -10,12 +10,10 @@ import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; import type { HostVm } from './types'; import type { BaseVmCreateOptions } from './vm_services'; -import { createMultipassHostVmClient, createVagrantHostVmClient, createVm } from './vm_services'; +import { createVm, getHostVmClient } from './vm_services'; import { downloadAndStoreAgent } from './agent_downloads_service'; import { enrollHostVmWithFleet, getAgentDownloadUrl, unEnrollFleetAgent } from './fleet_services'; -export const VAGRANT_CWD = `${__dirname}/../endpoint_agent_runner/`; - export interface CreateAndEnrollEndpointHostOptions extends Pick { kbnClient: KbnClient; @@ -111,10 +109,6 @@ export const destroyEndpointHost = async ( ]); }; -const getHostVmClient = (vmName: string): HostVm => { - return process.env.CI ? createVagrantHostVmClient(vmName) : createMultipassHostVmClient(vmName); -}; - export const deleteMultipassVm = async (vmName: string): Promise => { await getHostVmClient(vmName).destroy(); }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts index 77fc34ea8e4d4..38256f1c774bd 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/types.ts @@ -14,6 +14,8 @@ export interface HostVm { exec: (command: string) => Promise; mount: (localDir: string, hostVmDir: string) => Promise; unmount: (hostVmDir: string) => Promise; + /** Uploads/copies a file from the local machine to the VM */ + transfer: (localFilePath: string, destFilePath: string) => Promise; destroy: () => Promise; info: () => string; stop: () => void; @@ -30,3 +32,9 @@ export interface HostVmMountResponse { hostDir: string; unmount: () => Promise; } +export interface HostVmTransferResponse { + /** The file path of the file on the host vm */ + filePath: string; + /** Delete the file from the host VM */ + delete: () => Promise; +} diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index 5de708641eb07..a0efd908f80d7 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -16,7 +16,7 @@ import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils import type { HostVm, HostVmExecResponse, SupportedVmManager } from './types'; const baseGenerator = new BaseDataGenerator(); -const DEFAULT_VAGRANTFILE = pathJoin(__dirname, 'vagrant', 'Vagrantfile'); +export const DEFAULT_VAGRANTFILE = pathJoin(__dirname, 'vagrant', 'Vagrantfile'); export interface BaseVmCreateOptions { name: string; @@ -125,6 +125,20 @@ export const createMultipassHostVmClient = ( log.verbose(`multipass stop response:\n`, response); }; + const transfer: HostVm['transfer'] = async (localFilePath, destFilePath) => { + const response = await execa.command( + `multipass transfer ${localFilePath} ${name}:${destFilePath}` + ); + log.verbose(`Transferred file to VM [${name}]:`, response); + + return { + filePath: destFilePath, + delete: async () => { + return exec(`rm ${destFilePath}`); + }, + }; + }; + return { type: 'multipass', name, @@ -133,6 +147,7 @@ export const createMultipassHostVmClient = ( info, mount, unmount, + transfer, start, stop, }; @@ -315,6 +330,21 @@ export const createVagrantHostVmClient = ( log.verbose('vagrant suspend response:\n', response); }; + const transfer: HostVm['transfer'] = async (localFilePath, destFilePath) => { + const response = await execa.command( + `vagrant upload ${localFilePath} ${destFilePath}`, + execaOptions + ); + log.verbose(`Transferred file to VM [${name}]:`, response); + + return { + filePath: destFilePath, + delete: async () => { + return exec(`rm ${destFilePath}`); + }, + }; + }; + return { type: 'vagrant', name, @@ -323,7 +353,27 @@ export const createVagrantHostVmClient = ( info, mount, unmount, + transfer, start, stop, }; }; + +/** + * create and return a Host VM client client + * @param hostname + * @param type + * @param vagrantFile + * @param log + */ +export const getHostVmClient = ( + hostname: string, + type: SupportedVmManager = process.env.CI ? 'vagrant' : 'multipass', + /** Will only be used if `type` is `vagrant` */ + vagrantFile: string = DEFAULT_VAGRANTFILE, + log: ToolingLog = createToolingLogger() +): HostVm => { + return type === 'vagrant' + ? createVagrantHostVmClient(hostname, vagrantFile, log) + : createMultipassHostVmClient(hostname, log); +}; From 343568f25be8a26079870773edf13b2672db8189 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 6 Nov 2023 17:23:36 -0500 Subject: [PATCH 35/72] Add ability for `cy.waitUntil()` to also accept a message for logging --- .../public/management/cypress/cypress.d.ts | 9 +++++++++ .../public/management/cypress/support/e2e.ts | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts index bd4c34b36de59..bf7247781519f 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts @@ -84,9 +84,18 @@ declare global { /** * Continuously call provided callback function until it either return `true` * or fail if `timeout` is reached. + * @param message * @param fn * @param options */ + waitUntil( + message: string, + fn: (subject?: any) => boolean | Promise | Chainable, + options?: Partial<{ + interval: number; + timeout: number; + }> + ): Chainable; waitUntil( fn: (subject?: any) => boolean | Promise | Chainable, options?: Partial<{ diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts b/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts index e3f1c084843fe..700401f4ead0d 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts @@ -65,7 +65,12 @@ Cypress.Commands.addQuery<'findByTestSubj'>( Cypress.Commands.add( 'waitUntil', { prevSubject: 'optional' }, - (subject, fn, { interval = 500, timeout = 30000 } = {}) => { + (subject, msgOrFn, fnOrOptions, maybeOptions) => { + const [msg, fn, options] = + typeof msgOrFn === 'function' + ? ['', msgOrFn, fnOrOptions ?? {}] + : [msgOrFn, fnOrOptions, maybeOptions ?? {}]; + const { interval = 500, timeout = 30000 } = options; let attempts = Math.floor(timeout / interval); const completeOrRetry = (result: boolean) => { @@ -73,7 +78,7 @@ Cypress.Commands.add( return result; } if (attempts < 1) { - throw new Error(`Timed out while retrying, last result was: {${result}}`); + throw new Error(`${msg}: Timed out while retrying - last result was: {${result}}`); } cy.wait(interval, { log: false }).then(() => { attempts--; @@ -91,7 +96,7 @@ Cypress.Commands.add( return result.then(completeOrRetry); } else { throw new Error( - `Unknown return type from callback: ${Object.prototype.toString.call(result)}` + `${msg}: Unknown return type from callback: ${Object.prototype.toString.call(result)}` ); } }; From 4bffb1c2f6fb4fb02c5bb0cdcc633b3cc5c52002 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 6 Nov 2023 17:23:57 -0500 Subject: [PATCH 36/72] some additional logging for debug --- .../public/management/cypress/tasks/fleet.ts | 16 +++++++++++----- .../common/fleet_server/fleet_server_services.ts | 7 ++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts index 8f1da4a0ec020..58e46929ab5d5 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts @@ -203,8 +203,10 @@ const waitForHasAgentPolicyChanged = ( policyRevision: number ): Cypress.Chainable => { let isPolicyUpdated = false; + return cy .waitUntil( + `Wait for Fleet Agent to report policy revision ${policyRevision}`, () => { return request({ method: 'GET', @@ -213,16 +215,20 @@ const waitForHasAgentPolicyChanged = ( 'elastic-api-version': API_VERSIONS.public.v1, }, }).then((response) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { status, policy_revision, policy_id } = response.body.item; + + cy.log('Checking Agent data:', { status, policy_revision, policy_id }); + if ( - response.body.item.status !== 'updating' && - response.body.item?.policy_revision === policyRevision && - response.body.item?.policy_id === policyId + status !== 'updating' && + policy_revision === policyRevision && + policy_id === policyId ) { isPolicyUpdated = true; - return true; } - return false; + return cy.wrap(isPolicyUpdated); }); }, { timeout: 120000 } diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index ed1d478fe69fb..d91b269ab37bc 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -289,7 +289,7 @@ const startFleetServerWithDocker = async ({ await execa('docker', ['kill', containerName]) .then(() => { - log.verbose( + log.debug( `Killed an existing container with name [${containerName}]. New one will be started.` ); }) @@ -656,12 +656,13 @@ export const isFleetServerRunning = async ( httpsAgent: new https.Agent({ rejectUnauthorized: false }), }) .then((response) => { - log.debug(`Fleet server is up and running as [${fleetServerUrl}]`, response.data); + log.debug(`Fleet server is up and running at [${fleetServerUrl}]. Status: `, response.data); return true; }) .catch(catchAxiosErrorFormatAndThrow) .catch((e) => { - log.debug(`Fleet server not up. Attempt to call [${url.toString()}] failed with:`, e); + log.debug(`Fleet server not up at [${fleetServerUrl}]`); + log.verbose(`Call to [${url.toString()}] failed with:`, e); return false; }); }; From 0829eef949faf7e65fd1385ccdd960835ab8f4d3 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 6 Nov 2023 18:26:04 -0500 Subject: [PATCH 37/72] correct log text --- .../plugins/security_solution/scripts/run_cypress/parallel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 3540529c62de6..946f7c1f6d2fb 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -357,7 +357,7 @@ ${JSON.stringify( // Setup fleet if Cypress config requires it let fleetServer: void | StartedFleetServer; if (cypressConfigFile.env?.WITH_FLEET_SERVER) { - log.info(`Setting fleet-server for this Cypress config`); + log.info(`Setting up fleet-server for this Cypress config`); const kbnClient = createKbnClient({ url: baseUrl, From e1d3f998e67d51146f48976b190f4d8ad8e82027 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 08:42:55 -0500 Subject: [PATCH 38/72] adjustments to `waitUntil()` --- .../public/management/cypress/cypress.d.ts | 13 +++---------- .../public/management/cypress/support/e2e.ts | 9 ++------- .../public/management/cypress/tasks/fleet.ts | 6 ++---- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts index bf7247781519f..560d7b7c63146 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts @@ -84,24 +84,17 @@ declare global { /** * Continuously call provided callback function until it either return `true` * or fail if `timeout` is reached. - * @param message * @param fn * @param options + * @param message */ - waitUntil( - message: string, - fn: (subject?: any) => boolean | Promise | Chainable, - options?: Partial<{ - interval: number; - timeout: number; - }> - ): Chainable; waitUntil( fn: (subject?: any) => boolean | Promise | Chainable, options?: Partial<{ interval: number; timeout: number; - }> + }>, + message?: string ): Chainable; task( diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts b/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts index 700401f4ead0d..e7205052bfdf0 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/e2e.ts @@ -65,12 +65,7 @@ Cypress.Commands.addQuery<'findByTestSubj'>( Cypress.Commands.add( 'waitUntil', { prevSubject: 'optional' }, - (subject, msgOrFn, fnOrOptions, maybeOptions) => { - const [msg, fn, options] = - typeof msgOrFn === 'function' - ? ['', msgOrFn, fnOrOptions ?? {}] - : [msgOrFn, fnOrOptions, maybeOptions ?? {}]; - const { interval = 500, timeout = 30000 } = options; + (subject, fn, { interval = 500, timeout = 30000 } = {}, msg = 'waitUntil()') => { let attempts = Math.floor(timeout / interval); const completeOrRetry = (result: boolean) => { @@ -78,7 +73,7 @@ Cypress.Commands.add( return result; } if (attempts < 1) { - throw new Error(`${msg}: Timed out while retrying - last result was: {${result}}`); + throw new Error(`${msg}: Timed out while retrying - last result was: [${result}]`); } cy.wait(interval, { log: false }).then(() => { attempts--; diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts index 58e46929ab5d5..1a91d6154eecd 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts @@ -206,7 +206,6 @@ const waitForHasAgentPolicyChanged = ( return cy .waitUntil( - `Wait for Fleet Agent to report policy revision ${policyRevision}`, () => { return request({ method: 'GET', @@ -218,8 +217,6 @@ const waitForHasAgentPolicyChanged = ( // eslint-disable-next-line @typescript-eslint/naming-convention const { status, policy_revision, policy_id } = response.body.item; - cy.log('Checking Agent data:', { status, policy_revision, policy_id }); - if ( status !== 'updating' && policy_revision === policyRevision && @@ -231,7 +228,8 @@ const waitForHasAgentPolicyChanged = ( return cy.wrap(isPolicyUpdated); }); }, - { timeout: 120000 } + { timeout: 120000 }, + `Wait for Fleet Agent to report policy id [${policyId}] with revision [${policyRevision}]` ) .then(() => { return isPolicyUpdated; From ce991fd94fb9374bf8785bcbfb05eac9703be7af Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 09:39:35 -0500 Subject: [PATCH 39/72] added `logger` task ++ remove fleet server setup cy support file --- .../public/management/cypress/cypress.d.ts | 7 +++++ .../cypress/support/data_loaders.ts | 9 ++++++ .../cypress/support/fleet_server_setup.ts | 29 ------------------- .../public/management/cypress/tasks/fleet.ts | 3 ++ .../public/management/cypress/tasks/logger.ts | 19 ++++++++++++ .../public/management/cypress/types.ts | 6 ++++ .../scripts/run_cypress/parallel.ts | 7 +++-- 7 files changed, 49 insertions(+), 31 deletions(-) delete mode 100644 x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts create mode 100644 x-pack/plugins/security_solution/public/management/cypress/tasks/logger.ts diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts index 560d7b7c63146..0ae9db14cbce3 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts @@ -24,6 +24,7 @@ import type { CreateUserAndRoleCyTaskOptions, UninstallAgentFromHostTaskOptions, IsAgentAndEndpointUninstalledFromHostTaskOptions, + LogItTaskOptions, } from './types'; import type { DeleteIndexedFleetEndpointPoliciesResponse, @@ -219,6 +220,12 @@ declare global { arg: IsAgentAndEndpointUninstalledFromHostTaskOptions, options?: Partial ): Chainable; + + task( + name: 'logIt', + arg: LogItTaskOptions, + options?: Partial + ): Chainable; } } } diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index dc54c6e4db710..99ea877053c91 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -46,6 +46,7 @@ import type { IndexEndpointHostsCyTaskOptions, LoadUserAndRoleCyTaskOptions, CreateUserAndRoleCyTaskOptions, + LogItTaskOptions, } from '../types'; import type { DeletedIndexedEndpointRuleAlerts, @@ -134,6 +135,14 @@ export const dataLoaders = ( ); on('task', { + logIt: async ({ level = 'info', data }: LogItTaskOptions): Promise => { + return stackServicesPromise + .then(({ log }) => { + log[level](data); + }) + .then(() => null); + }, + indexFleetEndpointPolicy: async ({ policyName, endpointPackageVersion, diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts b/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts deleted file mode 100644 index 802c7b28061cf..0000000000000 --- a/x-pack/plugins/security_solution/public/management/cypress/support/fleet_server_setup.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { startFleetServerIfNecessary } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; -import { setupStackServicesUsingCypressConfig } from './common'; - -// FIXME:PT delete this - -export const setupFleetServerForCypressTestRun = async ( - on: Cypress.PluginEvents, - config: Cypress.PluginConfigOptions -) => { - const { kbnClient, log } = await setupStackServicesUsingCypressConfig(config); - - const startedFleetServer = await startFleetServerIfNecessary({ - kbnClient, - logger: log, - }).catch(log.error); - - on('after:run', async () => { - if (startedFleetServer) { - await startedFleetServer.stop(); - } - }); -}; diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts index 1a91d6154eecd..7281778d8ae68 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts @@ -26,6 +26,7 @@ import type { import { uninstallTokensRouteService } from '@kbn/fleet-plugin/common/services/routes'; import type { GetUninstallTokensMetadataResponse } from '@kbn/fleet-plugin/common/types/rest_spec/uninstall_token'; import type { UninstallToken } from '@kbn/fleet-plugin/common/types/models/uninstall_token'; +import { logger } from './logger'; import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; import { request } from './common'; @@ -217,6 +218,8 @@ const waitForHasAgentPolicyChanged = ( // eslint-disable-next-line @typescript-eslint/naming-convention const { status, policy_revision, policy_id } = response.body.item; + logger.debug('Checking policy data:', { status, policy_revision, policy_id }); + if ( status !== 'updating' && policy_revision === policyRevision && diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/logger.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/logger.ts new file mode 100644 index 0000000000000..053ee123c5954 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/logger.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/no-explicit-any */ +export const logger = Object.freeze({ + info: (...data: any): Cypress.Chainable => { + return cy.task('logIt', { level: 'info', data }); + }, + debug: (...data: any): Cypress.Chainable => { + return cy.task('logIt', { level: 'info', data }); + }, + verbose: (...data: any): Cypress.Chainable => { + return cy.task('logIt', { level: 'info', data }); + }, +}); diff --git a/x-pack/plugins/security_solution/public/management/cypress/types.ts b/x-pack/plugins/security_solution/public/management/cypress/types.ts index 6c5dae16100de..8beb150a64d5a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/types.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/types.ts @@ -8,6 +8,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { Role } from '@kbn/security-plugin/common'; +import type { ToolingLog } from '@kbn/tooling-log'; import type { ActionDetails } from '../../../common/endpoint/types'; import type { CyLoadEndpointDataOptions } from './support/plugin_handlers/endpoint_data_loader'; import type { SecurityTestUser } from './common/constants'; @@ -75,3 +76,8 @@ export interface UninstallAgentFromHostTaskOptions { export interface IsAgentAndEndpointUninstalledFromHostTaskOptions { hostname: string; } + +export interface LogItTaskOptions { + level: keyof Pick; + data: any; +} diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 946f7c1f6d2fb..d43ec6e26f8ef 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -29,7 +29,7 @@ import pRetry from 'p-retry'; import { createToolingLogger } from '../../common/endpoint/data_loaders/utils'; import { createKbnClient } from '../endpoint/common/stack_services'; import type { StartedFleetServer } from '../endpoint/common/fleet_server/fleet_server_services'; -import { startFleetServerIfNecessary } from '../endpoint/common/fleet_server/fleet_server_services'; +import { startFleetServer } from '../endpoint/common/fleet_server/fleet_server_services'; import { renderSummaryTable } from './print_run'; import { isSkipped, parseTestFileConfig } from './utils'; import { getFTRConfig } from './get_ftr_config'; @@ -366,12 +366,15 @@ ${JSON.stringify( log, }); - fleetServer = await startFleetServerIfNecessary({ + fleetServer = await startFleetServer({ kbnClient, logger: log, port: config.has('servers.fleetserver.port') ? (config.get('servers.fleetserver.port') as number) : undefined, + // `force` is needed to ensure that any currently running fleet server (perhaps left + // over from an interrupted run) is killed and a new one restarted + force: true, }); } From 7eb7126104658cb98c6d0afa0db36e62dd57b977 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 12:05:32 -0500 Subject: [PATCH 40/72] update tests for tamper protection --- ...ging_policy_from_disabled_to_enabled.cy.ts | 7 ++-- ...ging_policy_from_enabled_to_disabled.cy.ts | 4 +- ...nging_policy_from_enabled_to_enabled.cy.ts | 7 ++-- ...ging_policy_from_disabled_to_enabled.cy.ts | 7 ++-- ...ging_policy_from_enabled_to_disabled.cy.ts | 4 +- ...nging_policy_from_enabled_to_enabled.cy.ts | 7 ++-- .../public/management/cypress/tasks/fleet.ts | 42 ++++++++++++------- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts index d30345d8d5486..3d92528c2eee7 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts @@ -14,7 +14,7 @@ import { createAgentPolicyTask, enableAgentTamperProtectionFeatureFlagInPolicy, unenrollAgent, - changeAgentPolicy, + reAssignFleetAgentToPolicy, } from '../../../tasks/fleet'; import { login } from '../../../tasks/login'; @@ -79,10 +79,9 @@ describe( it('should unenroll from fleet without issues', () => { waitForEndpointListPageToBeLoaded(createdHost.hostname); // Change agent policy and wait for action to be completed - changeAgentPolicy( + reAssignFleetAgentToPolicy( createdHost.agentId, - policyWithAgentTamperProtectionEnabled.policy_id, - 3 + policyWithAgentTamperProtectionEnabled.policy_id ).then((hasChanged) => { expect(hasChanged).to.eql(true); unenrollAgent(createdHost.agentId).then((isUnenrolled) => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts index 59c70d85118dd..a9508a13f719b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts @@ -14,7 +14,7 @@ import { createAgentPolicyTask, enableAgentTamperProtectionFeatureFlagInPolicy, unenrollAgent, - changeAgentPolicy, + reAssignFleetAgentToPolicy, } from '../../../tasks/fleet'; import { login } from '../../../tasks/login'; @@ -79,7 +79,7 @@ describe( it('should unenroll from fleet without issues', () => { waitForEndpointListPageToBeLoaded(createdHost.hostname); // Change agent policy and wait for action to be completed - changeAgentPolicy(createdHost.agentId, policy.policy_id, 3).then((hasChanged) => { + reAssignFleetAgentToPolicy(createdHost.agentId, policy.policy_id).then((hasChanged) => { expect(hasChanged).to.eql(true); unenrollAgent(createdHost.agentId).then((isUnenrolled) => { expect(isUnenrolled).to.eql(true); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts index 3a897bc544ea8..a5654734c15e4 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts @@ -14,7 +14,7 @@ import { createAgentPolicyTask, enableAgentTamperProtectionFeatureFlagInPolicy, unenrollAgent, - changeAgentPolicy, + reAssignFleetAgentToPolicy, } from '../../../tasks/fleet'; import { login } from '../../../tasks/login'; @@ -81,10 +81,9 @@ describe( it('should unenroll from fleet without issues', () => { waitForEndpointListPageToBeLoaded(createdHost.hostname); // Change agent policy and wait for action to be completed - changeAgentPolicy( + reAssignFleetAgentToPolicy( createdHost.agentId, - secondPolicyWithAgentTamperProtectionEnabled.policy_id, - 3 + secondPolicyWithAgentTamperProtectionEnabled.policy_id ).then((hasChanged) => { expect(hasChanged).to.eql(true); unenrollAgent(createdHost.agentId).then((isUnenrolled) => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts index 5950288f2313e..bbb675cf56d5e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts @@ -14,7 +14,7 @@ import { createAgentPolicyTask, enableAgentTamperProtectionFeatureFlagInPolicy, getUninstallToken, - changeAgentPolicy, + reAssignFleetAgentToPolicy, isAgentAndEndpointUninstalledFromHost, uninstallAgentFromHost, } from '../../../tasks/fleet'; @@ -82,10 +82,9 @@ describe( waitForEndpointListPageToBeLoaded(createdHost.hostname); // Change agent policy and wait for action to be completed - changeAgentPolicy( + reAssignFleetAgentToPolicy( createdHost.agentId, - policyWithAgentTamperProtectionEnabled.policy_id, - 3 + policyWithAgentTamperProtectionEnabled.policy_id ).then((hasChanged) => { expect(hasChanged).to.eql(true); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts index e55872351aef0..0768c4a49ca39 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts @@ -13,7 +13,7 @@ import { getEndpointIntegrationVersion, createAgentPolicyTask, enableAgentTamperProtectionFeatureFlagInPolicy, - changeAgentPolicy, + reAssignFleetAgentToPolicy, isAgentAndEndpointUninstalledFromHost, uninstallAgentFromHost, } from '../../../tasks/fleet'; @@ -80,7 +80,7 @@ describe( it('should uninstall from host without issues', () => { waitForEndpointListPageToBeLoaded(createdHost.hostname); - changeAgentPolicy(createdHost.agentId, policy.policy_id, 3).then((hasChanged) => { + reAssignFleetAgentToPolicy(createdHost.agentId, policy.policy_id).then((hasChanged) => { expect(hasChanged).to.eql(true); uninstallAgentFromHost(createdHost.hostname).then((responseWithoutToken) => { expect(responseWithoutToken).to.not.match(/(.*)Invalid uninstall token(.*)/); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts index 15fd02ad14511..d8630a50a83b9 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts @@ -14,7 +14,7 @@ import { createAgentPolicyTask, enableAgentTamperProtectionFeatureFlagInPolicy, getUninstallToken, - changeAgentPolicy, + reAssignFleetAgentToPolicy, isAgentAndEndpointUninstalledFromHost, uninstallAgentFromHost, } from '../../../tasks/fleet'; @@ -85,10 +85,9 @@ describe( waitForEndpointListPageToBeLoaded(createdHost.hostname); // Change agent policy and wait for action to be completed - changeAgentPolicy( + reAssignFleetAgentToPolicy( createdHost.agentId, - secondPolicyWithAgentTamperProtectionEnabled.policy_id, - 3 + secondPolicyWithAgentTamperProtectionEnabled.policy_id ).then((hasChanged) => { expect(hasChanged).to.eql(true); diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts index 7281778d8ae68..a16f289ec8bbb 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/fleet.ts @@ -10,6 +10,7 @@ import type { GetAgentsResponse, GetInfoResponse, GetPackagePoliciesResponse, + GetOneAgentPolicyResponse, } from '@kbn/fleet-plugin/common'; import { agentRouteService, @@ -134,21 +135,33 @@ export const unenrollAgent = (agentId: string): Cypress.Chainable => { }); }; -export const changeAgentPolicy = ( +export const fetchFleetAgentPolicy = ( + agentPolicyId: string +): Cypress.Chainable => { + return request({ + method: 'GET', + url: agentPolicyRouteService.getInfoPath(agentPolicyId), + }).then((res) => res.body.item); +}; + +export const reAssignFleetAgentToPolicy = ( agentId: string, - policyId: string, - policyRevision: number + policyId: string ): Cypress.Chainable => { - return request({ - method: 'POST', - url: agentRouteService.getReassignPath(agentId), - body: { - policy_id: policyId, - }, - headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 }, - }).then(() => { - return waitForHasAgentPolicyChanged(agentId, policyId, policyRevision); - }); + return fetchFleetAgentPolicy(policyId) + .then((agentPolicy) => { + return request({ + method: 'POST', + url: agentRouteService.getReassignPath(agentId), + body: { + policy_id: policyId, + }, + headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 }, + }).then(() => agentPolicy); + }) + .then((agentPolicy) => { + return waitForHasAgentPolicyChanged(agentId, policyId, agentPolicy.revision); + }); }; // only used in "real" endpoint tests not in mocked ones @@ -201,6 +214,7 @@ const waitForIsAgentUnenrolled = (agentId: string): Cypress.Chainable = const waitForHasAgentPolicyChanged = ( agentId: string, policyId: string, + /** The minimum revision number that the agent must report before it is considered "changed" */ policyRevision: number ): Cypress.Chainable => { let isPolicyUpdated = false; @@ -222,7 +236,7 @@ const waitForHasAgentPolicyChanged = ( if ( status !== 'updating' && - policy_revision === policyRevision && + (policy_revision ?? 0) >= policyRevision && policy_id === policyId ) { isPolicyUpdated = true; From 03d99c315aaa4e0c82b99a14a1b76f8646b866ef Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 13:49:07 -0500 Subject: [PATCH 41/72] add log of existing agent policies to --- x-pack/test/osquery_cypress/utils.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 810f432d14dd9..d623321d793e6 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -42,6 +42,12 @@ export const createAgentPolicy = async ( ) => { log.info(`Creating "${agentPolicyName}" agent policy`); + // FIXME:PT Delete. only here for debug + const existing = await kbnClient.request({ method: 'GET', path: `/api/fleet/agent_policies` }); + Error.captureStackTrace(existing); + log.info(JSON.stringify(existing)); + // FIXME:PT delete + const { data: { item: { id: agentPolicyId }, From a7dd5d50b492db9f53eb92196c8d8c38d8b1435b Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 14:30:55 -0500 Subject: [PATCH 42/72] more debug changes for osquery --- x-pack/test/osquery_cypress/utils.ts | 40 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index d623321d793e6..61e598544f4b8 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -43,7 +43,12 @@ export const createAgentPolicy = async ( log.info(`Creating "${agentPolicyName}" agent policy`); // FIXME:PT Delete. only here for debug - const existing = await kbnClient.request({ method: 'GET', path: `/api/fleet/agent_policies` }); + const existing = await kbnClient + .request({ method: 'GET', path: `/api/fleet/agent_policies` }) + .catch((e) => { + Error.captureStackTrace(e); + throw e; + }); Error.captureStackTrace(existing); log.info(JSON.stringify(existing)); // FIXME:PT delete @@ -52,20 +57,25 @@ export const createAgentPolicy = async ( data: { item: { id: agentPolicyId }, }, - } = await kbnClient.request({ - method: 'POST', - path: `/api/fleet/agent_policies?sys_monitoring=true`, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: { - name: agentPolicyName, - description: '', - namespace: 'default', - monitoring_enabled: ['logs', 'metrics'], - inactivity_timeout: 1209600, - }, - }); + } = await kbnClient + .request({ + method: 'POST', + path: `/api/fleet/agent_policies?sys_monitoring=true`, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + body: { + name: agentPolicyName, + description: '', + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + inactivity_timeout: 1209600, + }, + }) + .catch((e) => { + Error.captureStackTrace(e); + throw e; + }); log.info(`Adding integration to ${agentPolicyId}`); From f0048a6064c5edf98b928ded29ed48380cf32b41 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 14:49:21 -0500 Subject: [PATCH 43/72] osquery - again --- x-pack/test/osquery_cypress/utils.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 61e598544f4b8..7c55026ed1825 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -44,7 +44,13 @@ export const createAgentPolicy = async ( // FIXME:PT Delete. only here for debug const existing = await kbnClient - .request({ method: 'GET', path: `/api/fleet/agent_policies` }) + .request({ + method: 'GET', + path: `/api/fleet/agent_policies`, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + }) .catch((e) => { Error.captureStackTrace(e); throw e; From d0a1a9591c63da23b88c10805110a788fb2c1e6d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 15:07:07 -0500 Subject: [PATCH 44/72] osquery - again --- x-pack/test/osquery_cypress/utils.ts | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 7c55026ed1825..f8b4de6e97770 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -43,20 +43,16 @@ export const createAgentPolicy = async ( log.info(`Creating "${agentPolicyName}" agent policy`); // FIXME:PT Delete. only here for debug - const existing = await kbnClient - .request({ - method: 'GET', - path: `/api/fleet/agent_policies`, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - }) - .catch((e) => { - Error.captureStackTrace(e); - throw e; - }); - Error.captureStackTrace(existing); - log.info(JSON.stringify(existing)); + const stacktrace = {}; + Error.captureStackTrace(stacktrace); + const existing = await kbnClient.request({ + method: 'GET', + path: `/api/fleet/agent_policies`, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + }); + log.info(`called from: `, JSON.stringify(stacktrace, null, 2), 'data:', JSON.stringify(existing)); // FIXME:PT delete const { From 04098a86c15005a8fbe480ff273fb1e629397f57 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 15:31:07 -0500 Subject: [PATCH 45/72] adjust osquery debug data --- x-pack/test/osquery_cypress/utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index f8b4de6e97770..97dfce7d55f29 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -52,7 +52,12 @@ export const createAgentPolicy = async ( 'elastic-api-version': API_VERSIONS.public.v1, }, }); - log.info(`called from: `, JSON.stringify(stacktrace, null, 2), 'data:', JSON.stringify(existing)); + log.info( + `called from: `, + JSON.stringify(stacktrace, null, 2), + 'data:', + JSON.stringify(existing.data) + ); // FIXME:PT delete const { From 4a38d1c869f12d232df93f221c95e8367fa37127 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 15:51:22 -0500 Subject: [PATCH 46/72] osquery - capture stack of call to create policy --- x-pack/test/osquery_cypress/utils.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 97dfce7d55f29..7c7878fe19a0a 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -43,8 +43,7 @@ export const createAgentPolicy = async ( log.info(`Creating "${agentPolicyName}" agent policy`); // FIXME:PT Delete. only here for debug - const stacktrace = {}; - Error.captureStackTrace(stacktrace); + const stacktrace = new Error('foo'); const existing = await kbnClient.request({ method: 'GET', path: `/api/fleet/agent_policies`, @@ -52,12 +51,7 @@ export const createAgentPolicy = async ( 'elastic-api-version': API_VERSIONS.public.v1, }, }); - log.info( - `called from: `, - JSON.stringify(stacktrace, null, 2), - 'data:', - JSON.stringify(existing.data) - ); + log.info(`called from: `, stacktrace.stack, 'data:', JSON.stringify(existing.data)); // FIXME:PT delete const { From 03dd9b841ce2e53f2f7de85e16383f024ac15ac9 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 7 Nov 2023 16:33:01 -0500 Subject: [PATCH 47/72] osquery: ensure policy names are unique --- x-pack/test/osquery_cypress/runner.ts | 9 ++++-- x-pack/test/osquery_cypress/utils.ts | 45 +++++++++------------------ 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 7e7ac5e652fd7..0d63ba3635a28 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -45,8 +45,13 @@ async function setupFleetAgent({ getService }: FtrProviderContext) { await new FleetManager(log).setup(); - const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, 'Default policy'); - const policyEnrollmentKeyTwo = await createAgentPolicy(kbnClient, log, 'Osquery policy'); + const unique = Math.random().toString(32).substring(2, 6); + const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); + const policyEnrollmentKeyTwo = await createAgentPolicy( + kbnClient, + log, + `Osquery policy (${unique})` + ); const port = config.get('servers.fleetserver.port'); diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 7c7878fe19a0a..810f432d14dd9 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -42,41 +42,24 @@ export const createAgentPolicy = async ( ) => { log.info(`Creating "${agentPolicyName}" agent policy`); - // FIXME:PT Delete. only here for debug - const stacktrace = new Error('foo'); - const existing = await kbnClient.request({ - method: 'GET', - path: `/api/fleet/agent_policies`, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - }); - log.info(`called from: `, stacktrace.stack, 'data:', JSON.stringify(existing.data)); - // FIXME:PT delete - const { data: { item: { id: agentPolicyId }, }, - } = await kbnClient - .request({ - method: 'POST', - path: `/api/fleet/agent_policies?sys_monitoring=true`, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: { - name: agentPolicyName, - description: '', - namespace: 'default', - monitoring_enabled: ['logs', 'metrics'], - inactivity_timeout: 1209600, - }, - }) - .catch((e) => { - Error.captureStackTrace(e); - throw e; - }); + } = await kbnClient.request({ + method: 'POST', + path: `/api/fleet/agent_policies?sys_monitoring=true`, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + body: { + name: agentPolicyName, + description: '', + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + inactivity_timeout: 1209600, + }, + }); log.info(`Adding integration to ${agentPolicyId}`); From a2c74022e40fea468f11cbda41319dcdf68a11c1 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 8 Nov 2023 16:09:23 -0500 Subject: [PATCH 48/72] add `stopNow()` to StartedServer type --- .../endpoint/common/fleet_server/fleet_server_services.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index d91b269ab37bc..43396f38927f3 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -78,6 +78,8 @@ interface StartedServer { url: string; /** Stop server */ stop: () => Promise; + /** Stop server synchronously. Sometimes useful when called from nodeJS unexpected exits. */ + stopNow: () => void; /** Any information about the server */ info?: string; } @@ -363,6 +365,12 @@ Kill container: ${chalk.cyan(`docker kill ${containerId}`)} ); await execa('docker', ['kill', containerId]); }, + stopNow: () => { + log.info( + `Stopping (kill) fleet server. Container name [${containerName}] id [${containerId}]` + ); + execa.sync('docker', ['kill', containerId]); + }, }; }); From 355c07d03a8ec0dda778b94a3869dea6d118d799 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 8 Nov 2023 16:10:07 -0500 Subject: [PATCH 49/72] Change OSQuery to use common methods for starting fleet-server --- x-pack/test/osquery_cypress/fleet_server.ts | 35 ++++++++++++--------- x-pack/test/osquery_cypress/runner.ts | 26 ++------------- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/x-pack/test/osquery_cypress/fleet_server.ts b/x-pack/test/osquery_cypress/fleet_server.ts index f1fa7a174ae37..b3ea2277d71f3 100644 --- a/x-pack/test/osquery_cypress/fleet_server.ts +++ b/x-pack/test/osquery_cypress/fleet_server.ts @@ -6,40 +6,47 @@ */ import { ToolingLog } from '@kbn/tooling-log'; -import execa from 'execa'; -import { runFleetServerIfNeeded } from '@kbn/security-solution-plugin/scripts/endpoint/endpoint_agent_runner/fleet_server'; +import { KbnClient } from '@kbn/test'; +import { + StartedFleetServer, + startFleetServer, +} from '@kbn/security-solution-plugin/scripts/endpoint/common/fleet_server/fleet_server_services'; import { Manager } from './resource_manager'; +import { getLatestAvailableAgentVersion } from './utils'; export class FleetManager extends Manager { - private fleetContainerId?: string; - private log: ToolingLog; + private fleetServer: StartedFleetServer | undefined = undefined; - constructor(log: ToolingLog) { + constructor(private readonly kbnClient: KbnClient, private readonly log: ToolingLog) { super(); - this.log = log; } public async setup(): Promise { - const fleetServerConfig = await runFleetServerIfNeeded(); - - if (!fleetServerConfig) { - throw new Error('Fleet server config not found'); + const version = await getLatestAvailableAgentVersion(this.kbnClient); + this.fleetServer = await startFleetServer({ + kbnClient: this.kbnClient, + logger: this.log, + force: true, + version, + }); + + if (!this.fleetServer) { + throw new Error('Fleet server was not started'); } - - this.fleetContainerId = fleetServerConfig.fleetServerContainerId; } public cleanup() { super.cleanup(); this.log.info('Removing old fleet config'); - if (this.fleetContainerId) { + if (this.fleetServer) { this.log.info('Closing fleet process'); try { - execa.sync('docker', ['kill', this.fleetContainerId]); + this.fleetServer.stopNow(); } catch (err) { this.log.error('Error closing fleet server process'); + this.log.verbose(err); } this.log.info('Fleet server process closed'); } diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 0d63ba3635a28..429422d51ff0d 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -8,42 +8,20 @@ import Url from 'url'; import { verifyDockerInstalled, maybeCreateDockerNetwork } from '@kbn/es'; -import { startRuntimeServices } from '@kbn/security-solution-plugin/scripts/endpoint/endpoint_agent_runner/runtime'; import { FtrProviderContext } from './ftr_provider_context'; import { AgentManager } from './agent'; import { FleetManager } from './fleet_server'; -import { createAgentPolicy, getLatestAvailableAgentVersion } from './utils'; +import { createAgentPolicy } from './utils'; async function setupFleetAgent({ getService }: FtrProviderContext) { const log = getService('log'); const config = getService('config'); const kbnClient = getService('kibanaServer'); - const elasticUrl = Url.format(config.get('servers.elasticsearch')); - const kibanaUrl = Url.format(config.get('servers.kibana')); - const fleetServerUrl = Url.format({ - protocol: config.get('servers.kibana.protocol'), - hostname: config.get('servers.kibana.hostname'), - port: config.get('servers.fleetserver.port'), - }); - const username = config.get('servers.elasticsearch.username'); - const password = config.get('servers.elasticsearch.password'); - await verifyDockerInstalled(log); await maybeCreateDockerNetwork(log); - - await startRuntimeServices({ - log, - elasticUrl, - kibanaUrl, - fleetServerUrl, - username, - password, - version: await getLatestAvailableAgentVersion(kbnClient), - }); - - await new FleetManager(log).setup(); + await new FleetManager(kbnClient, log).setup(); const unique = Math.random().toString(32).substring(2, 6); const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); From a804ddc920d0762cbd5ae88558f455007f94781e Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 8 Nov 2023 16:14:47 -0500 Subject: [PATCH 50/72] removed `fleet_server` from `endpoint_agent_runner` script --- .../endpoint_agent_runner/fleet_server.ts | 42 ------------------- .../endpoint/endpoint_agent_runner/setup.ts | 11 +++-- 2 files changed, 8 insertions(+), 45 deletions(-) delete mode 100644 x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts deleted file mode 100644 index e1e7ddc66595f..0000000000000 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - isFleetServerRunning, - startFleetServer, -} from '../common/fleet_server/fleet_server_services'; -import { getRuntimeServices } from './runtime'; - -/** - * @deprecated - */ -export const runFleetServerIfNeeded = async (): Promise< - { fleetServerContainerId: string; fleetServerAgentPolicyId: string | undefined } | undefined -> => { - // TODO:PT remove dependencies on this module and then delete this and just call the code below directly from the CLI tool - - const { log, kbnClient } = getRuntimeServices(); - - // Runs under CI should force fleet server to be installed because the CI process first - // setups up Fleet server URL in kibana and then expects the server to be started. Forcing - // install here will ensure the fleet server is setup even if it appears to already be setup - // for kibana. - const forceInstall = Boolean(process.env.CI); - - if (forceInstall || !(await isFleetServerRunning(kbnClient))) { - const startedFleetServer = await startFleetServer({ - kbnClient, - logger: log, - force: forceInstall, - }); - - return { - fleetServerContainerId: startedFleetServer.id, - fleetServerAgentPolicyId: startedFleetServer.policyId, - }; - } -}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/setup.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/setup.ts index 18ef51a35bcca..c851b6ee34dbe 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/setup.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/setup.ts @@ -5,18 +5,23 @@ * 2.0. */ -import { runFleetServerIfNeeded } from './fleet_server'; -import { startRuntimeServices, stopRuntimeServices } from './runtime'; +import { getRuntimeServices, startRuntimeServices, stopRuntimeServices } from './runtime'; import { checkDependencies } from './pre_check'; import { enrollEndpointHost } from './elastic_endpoint'; import type { StartRuntimeServicesOptions } from './types'; +import { startFleetServerIfNecessary } from '../common/fleet_server/fleet_server_services'; export const setupAll = async (options: StartRuntimeServicesOptions) => { await startRuntimeServices(options); + const { kbnClient, log } = getRuntimeServices(); + await checkDependencies(); - await runFleetServerIfNeeded(); + await startFleetServerIfNecessary({ + kbnClient, + logger: log, + }); await enrollEndpointHost(); From aa28313809970f6dbc8b97561ab26dbff4939232 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 8 Nov 2023 16:27:17 -0500 Subject: [PATCH 51/72] remove fleet server cli options from script --- .../scripts/endpoint/endpoint_agent_runner/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts index 5e596c0c51a76..e66c85a2d50a7 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts @@ -12,6 +12,7 @@ import { setupAll } from './setup'; const runSetupAll: RunFn = async (cliContext) => { const username = cliContext.flags.username as string; const password = cliContext.flags.password as string; + const apiKey = cliContext.flags.apiKey as string; const kibanaUrl = cliContext.flags.kibanaUrl as string; const elasticUrl = cliContext.flags.elasticUrl as string; const fleetServerUrl = cliContext.flags.fleetServerUrl as string; @@ -28,6 +29,7 @@ const runSetupAll: RunFn = async (cliContext) => { version, policy, log, + apiKey, }); }; @@ -47,7 +49,6 @@ export const cli = () => { default: { kibanaUrl: 'http://127.0.0.1:5601', elasticUrl: 'http://127.0.0.1:9200', - fleetServerUrl: 'https://127.0.0.1:8220', username: 'elastic', password: 'changeme', version: '', @@ -65,7 +66,6 @@ export const cli = () => { --password Optional. Password associated with the username (Default: changeme) --kibanaUrl Optional. The url to Kibana (Default: http://127.0.0.1:5601) --elasticUrl Optional. The url to Elasticsearch (Default: http://127.0.0.1:9200) - --fleetServerUrl Optional. The url to Fleet Server (Default: https://127.0.0.1:8220) `, }, } From 67e6200bbea04b7422451f4817cb4ffc0dc2fcd8 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 8 Nov 2023 16:53:32 -0500 Subject: [PATCH 52/72] remove poc code --- .../scripts/endpoint/endpoint_agent_runner/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts index e66c85a2d50a7..51fb0ea3b5148 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/index.ts @@ -12,7 +12,6 @@ import { setupAll } from './setup'; const runSetupAll: RunFn = async (cliContext) => { const username = cliContext.flags.username as string; const password = cliContext.flags.password as string; - const apiKey = cliContext.flags.apiKey as string; const kibanaUrl = cliContext.flags.kibanaUrl as string; const elasticUrl = cliContext.flags.elasticUrl as string; const fleetServerUrl = cliContext.flags.fleetServerUrl as string; @@ -29,7 +28,6 @@ const runSetupAll: RunFn = async (cliContext) => { version, policy, log, - apiKey, }); }; From 3df10008907815751f21fd2b37557a7a9de827d1 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 8 Nov 2023 19:40:16 -0500 Subject: [PATCH 53/72] increase timeout for waiting on agents to show up in fleet --- .../security_solution/scripts/endpoint/common/fleet_services.ts | 2 +- x-pack/test/osquery_cypress/agent.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 1f6efee2906b2..af6a48964cbea 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -197,7 +197,7 @@ export const waitForHostToEnroll = async ( throw Object.assign( new Error( `Timed out waiting for host [${hostname}] to show up in Fleet in ${ - timeoutMs / 60 / 1000 + timeoutMs / 1000 } seconds` ), { agentId, hostname } diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts index cd7969703c483..510a33d952996 100644 --- a/x-pack/test/osquery_cypress/agent.ts +++ b/x-pack/test/osquery_cypress/agent.ts @@ -65,7 +65,7 @@ export class AgentManager extends Manager { ]; this.agentContainerId = (await execa('docker', dockerArgs)).stdout; - await waitForHostToEnroll(this.kbnClient, containerName); + await waitForHostToEnroll(this.kbnClient, containerName, 240000); } public cleanup() { From 2ae9bfdbf40a2e0ce76ac866e818b4ff386e3027 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 08:24:41 -0500 Subject: [PATCH 54/72] capture docker container output after start --- x-pack/test/osquery_cypress/agent.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts index 510a33d952996..dc7bf061ed2a0 100644 --- a/x-pack/test/osquery_cypress/agent.ts +++ b/x-pack/test/osquery_cypress/agent.ts @@ -64,7 +64,13 @@ export class AgentManager extends Manager { artifact, ]; - this.agentContainerId = (await execa('docker', dockerArgs)).stdout; + this.log.info(`Docker args: \n${JSON.stringify(dockerArgs, null, 2)}`); + + const startedContainer = await execa('docker', dockerArgs); + + this.log.info(`started: ${JSON.stringify(startedContainer, null, 2)}`); + + this.agentContainerId = startedContainer.stdout; await waitForHostToEnroll(this.kbnClient, containerName, 240000); } From 7c3f40972181acc68a846680df3620acf8e58826 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 08:49:51 -0500 Subject: [PATCH 55/72] run all fleet serer containers flavors with network `elastic` --- .../endpoint/common/fleet_server/fleet_server_services.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 43396f38927f3..1cefcdd8a8d6c 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -264,12 +264,14 @@ const startFleetServerWithDocker = async ({ - version adjusted to [latest] from [${agentVersion}]`); agentVersion = 'latest'; - await maybeCreateDockerNetwork(log); } else { assert.ok(!!policyId, '`policyId` is required'); assert.ok(!!serviceToken, '`serviceToken` is required'); } + // Create the `elastic` network to use with all containers + await maybeCreateDockerNetwork(log); + try { const dockerArgs = isServerless ? getFleetServerStandAloneDockerArgs({ @@ -405,6 +407,9 @@ const getFleetServerManagedDockerArgs = ({ '--restart', 'no', + '--net', + 'elastic', + '--add-host', 'host.docker.internal:host-gateway', From 2e575fbac85bfb6bf7b2ef37424ef53d1a613fb8 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 11:07:00 -0500 Subject: [PATCH 56/72] add more debug output --- .../common/fleet_server/fleet_server_services.ts | 9 ++++++++- .../scripts/endpoint/common/fleet_services.ts | 2 +- x-pack/test/osquery_cypress/agent.ts | 4 +--- x-pack/test/osquery_cypress/runner.ts | 6 +++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 1cefcdd8a8d6c..7c795555c8218 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -120,6 +120,13 @@ export const startFleetServer = async ({ port = 8220, }: StartFleetServerOptions): Promise => { logger.info(`Starting Fleet Server and connecting it to Kibana`); + logger.debug( + `called from:\n${(() => { + const s = { stack: '' }; + Error.captureStackTrace(s); + return s.stack; + })()}` + ); return logger.indent(4, async () => { // Check if fleet already running if `force` is false @@ -293,7 +300,7 @@ const startFleetServerWithDocker = async ({ await execa('docker', ['kill', containerName]) .then(() => { - log.debug( + log.info( `Killed an existing container with name [${containerName}]. New one will be started.` ); }) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index af6a48964cbea..3402c38ff4ad1 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -196,7 +196,7 @@ export const waitForHostToEnroll = async ( if (!found) { throw Object.assign( new Error( - `Timed out waiting for host [${hostname}] to show up in Fleet in ${ + `Timed out waiting for host [${hostname}] to show up in Fleet. Waited ${ timeoutMs / 1000 } seconds` ), diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts index dc7bf061ed2a0..1c9add043605a 100644 --- a/x-pack/test/osquery_cypress/agent.ts +++ b/x-pack/test/osquery_cypress/agent.ts @@ -64,11 +64,9 @@ export class AgentManager extends Manager { artifact, ]; - this.log.info(`Docker args: \n${JSON.stringify(dockerArgs, null, 2)}`); - const startedContainer = await execa('docker', dockerArgs); - this.log.info(`started: ${JSON.stringify(startedContainer, null, 2)}`); + this.log.info(`agent docker container started:\n${JSON.stringify(startedContainer, null, 2)}`); this.agentContainerId = startedContainer.stdout; await waitForHostToEnroll(this.kbnClient, containerName, 240000); diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 429422d51ff0d..fb6fb19ba17c0 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -8,6 +8,7 @@ import Url from 'url'; import { verifyDockerInstalled, maybeCreateDockerNetwork } from '@kbn/es'; +import { createToolingLogger } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/utils'; import { FtrProviderContext } from './ftr_provider_context'; import { AgentManager } from './agent'; @@ -15,10 +16,13 @@ import { FleetManager } from './fleet_server'; import { createAgentPolicy } from './utils'; async function setupFleetAgent({ getService }: FtrProviderContext) { - const log = getService('log'); + // const log = getService('log'); const config = getService('config'); const kbnClient = getService('kibanaServer'); + createToolingLogger.defaultLogLevel = 'verbose'; + const log = createToolingLogger(); + await verifyDockerInstalled(log); await maybeCreateDockerNetwork(log); await new FleetManager(kbnClient, log).setup(); From c62087f0ffd03f4c9ff18abc9d18eff7bfe69a6c Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 12:26:27 -0500 Subject: [PATCH 57/72] fix bug in retrieving fleet server agent version + log where startFleetSerer() is called from --- .../fleet_server/fleet_server_services.ts | 19 ++++++++----------- .../scripts/endpoint/common/utils.ts | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index 7c795555c8218..fa7f82db9ea6a 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -42,6 +42,7 @@ import { import { maybeCreateDockerNetwork, SERVERLESS_NODES, verifyDockerInstalled } from '@kbn/es'; import { resolve } from 'path'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { captureCallingStack } from '../utils'; import { createToolingLogger, RETRYABLE_TRANSIENT_ERRORS, @@ -120,13 +121,7 @@ export const startFleetServer = async ({ port = 8220, }: StartFleetServerOptions): Promise => { logger.info(`Starting Fleet Server and connecting it to Kibana`); - logger.debug( - `called from:\n${(() => { - const s = { stack: '' }; - Error.captureStackTrace(s); - return s.stack; - })()}` - ); + logger.debug(captureCallingStack()); return logger.indent(4, async () => { // Check if fleet already running if `force` is false @@ -248,13 +243,15 @@ const startFleetServerWithDocker = async ({ await verifyDockerInstalled(log); let agentVersion = version || (await getAgentVersionMatchingCurrentStack(kbnClient)); + const localhostRealIp = getLocalhostRealIp(); + const fleetServerUrl = `https://${localhostRealIp}:${port}`; - log.info(`Starting a new fleet server using Docker (version: ${agentVersion})`); + log.info( + `Starting a new fleet server using Docker\n Agent version: ${agentVersion}\n Server URL: ${fleetServerUrl}` + ); const response: StartedServer = await log.indent(4, async () => { const isServerless = await isServerlessKibanaFlavor(kbnClient); - const localhostRealIp = getLocalhostRealIp(); - const fleetServerUrl = `https://${localhostRealIp}:${port}`; const esURL = new URL(await getFleetElasticsearchOutputHost(kbnClient)); const containerName = `dev-fleet-server.${port}`; const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`; @@ -341,7 +338,7 @@ const startFleetServerWithDocker = async ({ containerName, '/bin/bash', '-c', - './elastic-agent version', + '/usr/share/elastic-agent/elastic-agent version', ]).catch((err) => { log.verbose(`Failed to retrieve agent version information from running instance.`, err); return { stdout: 'Unable to retrieve version information' }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts new file mode 100644 index 0000000000000..2e75e8e39d590 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Capture and return the calling stack for the context that called this utility. + */ +export const captureCallingStack = () => { + const s = { stack: '' }; + Error.captureStackTrace(s); + return `Called from:\n${s.stack.split('\n').slice(3).join('\n')}`; +}; From 7eafcb214f84b59b3045a71b33ff85e5ba3fd2a5 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 12:29:06 -0500 Subject: [PATCH 58/72] Osquery: fetch fleet-server URL from stack for enrolling agents via docker --- x-pack/test/osquery_cypress/agent.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts index 1c9add043605a..4143e47a2f84e 100644 --- a/x-pack/test/osquery_cypress/agent.ts +++ b/x-pack/test/osquery_cypress/agent.ts @@ -8,7 +8,10 @@ import execa from 'execa'; import { ToolingLog } from '@kbn/tooling-log'; import { KbnClient } from '@kbn/test'; -import { waitForHostToEnroll } from '@kbn/security-solution-plugin/scripts/endpoint/common/fleet_services'; +import { + fetchFleetServerUrl, + waitForHostToEnroll, +} from '@kbn/security-solution-plugin/scripts/endpoint/common/fleet_services'; import { getLatestVersion } from './artifact_manager'; import { Manager } from './resource_manager'; @@ -40,6 +43,9 @@ export class AgentManager extends Manager { const artifact = `docker.elastic.co/beats/elastic-agent:${await getLatestVersion()}`; this.log.info(artifact); const containerName = generateRandomString(12); + const fleetServerUrl = + (await fetchFleetServerUrl(this.kbnClient)) ?? + `https://host.docker.internal:${this.fleetServerPort}`; const dockerArgs = [ 'run', @@ -55,7 +61,7 @@ export class AgentManager extends Manager { '--env', 'FLEET_ENROLL=1', '--env', - `FLEET_URL=https://host.docker.internal:${this.fleetServerPort}`, + `FLEET_URL=${fleetServerUrl}`, '--env', `FLEET_ENROLLMENT_TOKEN=${this.policyEnrollmentKey}`, '--env', From 666100a31970355ff1fe27c7dde8c0d85aeed3a8 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 14:51:13 -0500 Subject: [PATCH 59/72] add logging to `waitForHostToEnroll()` --- .../common/fleet_server/fleet_server_services.ts | 6 +----- .../scripts/endpoint/common/fleet_services.ts | 10 ++++++++-- x-pack/test/osquery_cypress/agent.ts | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index fa7f82db9ea6a..c88374b6f6fb0 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -325,11 +325,7 @@ const startFleetServerWithDocker = async ({ await waitForFleetServerToRegisterWithElasticsearch(kbnClient, hostname, 120000); } else { - log.info('Waiting for server to show up in Kibana Fleet'); - - const fleetServerAgent = await waitForHostToEnroll(kbnClient, hostname, 120000); - - log.verbose(`Fleet server enrolled agent:\n${JSON.stringify(fleetServerAgent, null, 2)}`); + await waitForHostToEnroll(kbnClient, log, hostname, 120000); } fleetServerVersionInfo = ( diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 3402c38ff4ad1..26ca9d6474393 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -157,14 +157,18 @@ export const fetchFleetAgents = async ( * Will keep querying Fleet list of agents until the given `hostname` shows up as healthy * * @param kbnClient + * @param log * @param hostname * @param timeoutMs */ export const waitForHostToEnroll = async ( kbnClient: KbnClient, + log: ToolingLog, hostname: string, timeoutMs: number = 30000 ): Promise => { + log.info(`Waiting for host [${hostname}] to enroll with fleet`); + const started = new Date(); const hasTimedOut = (): boolean => { const elapsedTime = Date.now() - started.getTime(); @@ -204,6 +208,9 @@ export const waitForHostToEnroll = async ( ); } + log.debug(`Host [${hostname}] has been enrolled with fleet`); + log.verbose(found); + return found; }; @@ -681,8 +688,7 @@ export const enrollHostVmWithFleet = async ({ await hostVm.exec(agentEnrollCommand); - log.info(`Waiting for Agent to check-in with Fleet`); - return waitForHostToEnroll(kbnClient, hostVm.name, timeoutMs); + return waitForHostToEnroll(kbnClient, log, hostVm.name, timeoutMs); }; interface GetOrCreateDefaultAgentPolicyOptions { diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts index 4143e47a2f84e..63429fb4e5b7e 100644 --- a/x-pack/test/osquery_cypress/agent.ts +++ b/x-pack/test/osquery_cypress/agent.ts @@ -75,7 +75,7 @@ export class AgentManager extends Manager { this.log.info(`agent docker container started:\n${JSON.stringify(startedContainer, null, 2)}`); this.agentContainerId = startedContainer.stdout; - await waitForHostToEnroll(this.kbnClient, containerName, 240000); + await waitForHostToEnroll(this.kbnClient, this.log, containerName, 240000); } public cleanup() { From b5d8b7ac391978f0c4167d0dbe4cbd4f3f3a3a41 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 15:51:20 -0500 Subject: [PATCH 60/72] OSQuery: revert back policy names --- x-pack/test/osquery_cypress/runner.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index fb6fb19ba17c0..66ba9cfe8405f 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -27,13 +27,18 @@ async function setupFleetAgent({ getService }: FtrProviderContext) { await maybeCreateDockerNetwork(log); await new FleetManager(kbnClient, log).setup(); - const unique = Math.random().toString(32).substring(2, 6); - const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); - const policyEnrollmentKeyTwo = await createAgentPolicy( - kbnClient, - log, - `Osquery policy (${unique})` - ); + // FIXME:PT clean up here and above + // + // const unique = Math.random().toString(32).substring(2, 6); + // const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); + // const policyEnrollmentKeyTwo = await createAgentPolicy( + // kbnClient, + // log, + // `Osquery policy (${unique})` + // ); + + const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, 'Default policy'); + const policyEnrollmentKeyTwo = await createAgentPolicy(kbnClient, log, 'Osquery policy'); const port = config.get('servers.fleetserver.port'); From de52d2e99b283d367e6c27e72b1b8155eb64d9a4 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 16:32:27 -0500 Subject: [PATCH 61/72] add prefixes to certain ToolingLog instances --- .../management/cypress/support/common.ts | 6 +++ .../endpoint/common/endpoint_host_services.ts | 4 +- .../fleet_server/fleet_server_services.ts | 6 ++- .../scripts/endpoint/common/utils.ts | 48 +++++++++++++++++++ .../scripts/run_cypress/parallel.ts | 3 +- x-pack/test/osquery_cypress/runner.ts | 7 ++- 6 files changed, 68 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/common.ts b/x-pack/plugins/security_solution/public/management/cypress/support/common.ts index 6b4c30b884f8b..c356536cc03d4 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/common.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/common.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { prefixedOutputLogger } from '../../../../scripts/endpoint/common/utils'; import type { RuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; import { createRuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; @@ -25,6 +26,11 @@ export const setupStackServicesUsingCypressConfig = async (config: Cypress.Plugi esUsername: config.env.ELASTICSEARCH_USERNAME, esPassword: config.env.ELASTICSEARCH_PASSWORD, asSuperuser: true, + }).then(({ log, ...others }) => { + return { + ...others, + log: prefixedOutputLogger('cy.dfw', log), + }; }); RUNTIME_SERVICES_CACHE.set(config, stackServices); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index b48f5d1309a01..ae86bb7cdf9cf 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -8,6 +8,7 @@ import { kibanaPackageJson } from '@kbn/repo-info'; import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; +import { prefixedOutputLogger } from './utils'; import type { HostVm } from './types'; import type { BaseVmCreateOptions } from './vm_services'; import { createVm, getHostVmClient } from './vm_services'; @@ -41,7 +42,7 @@ export interface CreateAndEnrollEndpointHostResponse { */ export const createAndEnrollEndpointHost = async ({ kbnClient, - log, + log: _log, agentPolicyId, cpus, disk, @@ -51,6 +52,7 @@ export const createAndEnrollEndpointHost = async ({ useClosestVersionMatch = false, useCache = true, }: CreateAndEnrollEndpointHostOptions): Promise => { + const log = prefixedOutputLogger('createAndEnrollEndpointHost()', _logger); const isRunningInCI = Boolean(process.env.CI); const vmName = hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`; const { url: agentUrl } = await getAgentDownloadUrl(version, useClosestVersionMatch, log); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index c88374b6f6fb0..d6944c74ac31b 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -42,7 +42,7 @@ import { import { maybeCreateDockerNetwork, SERVERLESS_NODES, verifyDockerInstalled } from '@kbn/es'; import { resolve } from 'path'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { captureCallingStack } from '../utils'; +import { captureCallingStack, prefixedOutputLogger } from '../utils'; import { createToolingLogger, RETRYABLE_TRANSIENT_ERRORS, @@ -114,12 +114,14 @@ export interface StartedFleetServer extends StartedServer { */ export const startFleetServer = async ({ kbnClient, - logger, + logger: _logger, policy, version, force = false, port = 8220, }: StartFleetServerOptions): Promise => { + const logger = prefixedOutputLogger('startFleetServer()', _logger); + logger.info(`Starting Fleet Server and connecting it to Kibana`); logger.debug(captureCallingStack()); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts index 2e75e8e39d590..3c8e227c89271 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/utils.ts @@ -5,6 +5,11 @@ * 2.0. */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import chalk from 'chalk'; + /** * Capture and return the calling stack for the context that called this utility. */ @@ -13,3 +18,46 @@ export const captureCallingStack = () => { Error.captureStackTrace(s); return `Called from:\n${s.stack.split('\n').slice(3).join('\n')}`; }; + +/** + * Returns a logger that intercepts calls to the ToolingLog instance methods passed in input + * and prefix it with the provided value. Useful in order to track log entries, especially when + * logging output from multiple sources is concurrently being output to the same source + * (ex. CI jobs and output to stdout). + * + * @param prefix + * @param log + * + * @example + * const logger = new ToolingLog(); + * const prefixedLogger = prefixedOutputLogger('my_log', logger); + * + * prefixedLogger.info('log something'); // => info [my_log] log something + */ +export const prefixedOutputLogger = (prefix: string, log: ToolingLog): ToolingLog => { + const styledPrefix = `[${chalk.grey(prefix)}]`; + const logIt = (type: keyof ToolingLog, ...args: any) => { + return log[type](styledPrefix, ...args); + }; + + const logger: Partial = { + info: logIt.bind(null, 'info'), + debug: logIt.bind(null, 'debug'), + verbose: logIt.bind(null, 'verbose'), + success: logIt.bind(null, 'success'), + warning: logIt.bind(null, 'warning'), + write: logIt.bind(null, 'write'), + }; + + const proxy = new Proxy(log, { + get(target: ToolingLog, prop: keyof ToolingLog, receiver: any): any { + if (prop in logger) { + return logger[prop]; + } + + return log[prop]; + }, + }); + + return proxy; +}; diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 689e691f4dab3..dd2b09bdb66b1 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -26,6 +26,7 @@ import { import { createFailError } from '@kbn/dev-cli-errors'; import pRetry from 'p-retry'; +import { prefixedOutputLogger } from '../endpoint/common/utils'; import { createToolingLogger } from '../../common/endpoint/data_loaders/utils'; import { createKbnClient } from '../endpoint/common/stack_services'; import type { StartedFleetServer } from '../endpoint/common/fleet_server/fleet_server_services'; @@ -71,7 +72,7 @@ ${JSON.stringify(argv, null, 2)} createToolingLogger.defaultLogLevel = cypressConfigFile.env.TOOLING_LOG_LEVEL; } - const log = createToolingLogger(); + const log = prefixedOutputLogger('cy.parallel()', createToolingLogger()); log.info(` ---------------------------------------------- diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 66ba9cfe8405f..b1ff9331326e2 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -9,6 +9,7 @@ import Url from 'url'; import { verifyDockerInstalled, maybeCreateDockerNetwork } from '@kbn/es'; import { createToolingLogger } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/utils'; +import { prefixedOutputLogger } from '@kbn/security-solution-plugin/scripts/endpoint/common/utils'; import { FtrProviderContext } from './ftr_provider_context'; import { AgentManager } from './agent'; @@ -16,12 +17,14 @@ import { FleetManager } from './fleet_server'; import { createAgentPolicy } from './utils'; async function setupFleetAgent({ getService }: FtrProviderContext) { + // Un-comment line below to set tooling log levels to verbose. Useful when debugging + createToolingLogger.defaultLogLevel = 'verbose'; + // const log = getService('log'); const config = getService('config'); const kbnClient = getService('kibanaServer'); - createToolingLogger.defaultLogLevel = 'verbose'; - const log = createToolingLogger(); + const log = prefixedOutputLogger('cy.OSQuery', createToolingLogger()); await verifyDockerInstalled(log); await maybeCreateDockerNetwork(log); From 54fe3d02b81c2a77f89184e88a750c75cb38cd49 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 17:01:38 -0500 Subject: [PATCH 62/72] osquery: reuse policy if it already exists` --- x-pack/test/osquery_cypress/utils.ts | 48 ++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 810f432d14dd9..334f64a38d0d9 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -8,13 +8,19 @@ import axios from 'axios'; import semver from 'semver'; import { map } from 'lodash'; -import { PackagePolicy, CreatePackagePolicyResponse, API_VERSIONS } from '@kbn/fleet-plugin/common'; +import { + PackagePolicy, + CreatePackagePolicyResponse, + API_VERSIONS, + AGENT_POLICY_SAVED_OBJECT_TYPE, +} from '@kbn/fleet-plugin/common'; import { KbnClient } from '@kbn/test'; import { GetEnrollmentAPIKeysResponse, CreateAgentPolicyResponse, } from '@kbn/fleet-plugin/common/types'; import { ToolingLog } from '@kbn/tooling-log'; +import { fetchAgentPolicyList } from '@kbn/security-solution-plugin/scripts/endpoint/common/fleet_services'; export const DEFAULT_HEADERS = Object.freeze({ 'x-elastic-internal-product': 'security-solution', @@ -46,19 +52,33 @@ export const createAgentPolicy = async ( data: { item: { id: agentPolicyId }, }, - } = await kbnClient.request({ - method: 'POST', - path: `/api/fleet/agent_policies?sys_monitoring=true`, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: { - name: agentPolicyName, - description: '', - namespace: 'default', - monitoring_enabled: ['logs', 'metrics'], - inactivity_timeout: 1209600, - }, + } = await fetchAgentPolicyList(kbnClient, { + kuery: `${AGENT_POLICY_SAVED_OBJECT_TYPE}.name: "${agentPolicyName}"`, + }).then((response) => { + if (response.items[0]) { + log.debug(`Policy with name [${agentPolicyName}] already exists. Re-using it`); + + return { + data: { + item: response.items[0], + }, + }; + } + + return kbnClient.request({ + method: 'POST', + path: `/api/fleet/agent_policies?sys_monitoring=true`, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + body: { + name: agentPolicyName, + description: '', + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + inactivity_timeout: 1209600, + }, + }); }); log.info(`Adding integration to ${agentPolicyId}`); From 25820279ce806f278008ce8edffaae6f59afe956 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 17:08:46 -0500 Subject: [PATCH 63/72] fix var name --- .../scripts/endpoint/common/endpoint_host_services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index ae86bb7cdf9cf..cb2718e72ba26 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -52,7 +52,7 @@ export const createAndEnrollEndpointHost = async ({ useClosestVersionMatch = false, useCache = true, }: CreateAndEnrollEndpointHostOptions): Promise => { - const log = prefixedOutputLogger('createAndEnrollEndpointHost()', _logger); + const log = prefixedOutputLogger('createAndEnrollEndpointHost()', _log); const isRunningInCI = Boolean(process.env.CI); const vmName = hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`; const { url: agentUrl } = await getAgentDownloadUrl(version, useClosestVersionMatch, log); From e50de72b339d6893f1d61e46c5059ab21642dad4 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Thu, 9 Nov 2023 18:08:04 -0500 Subject: [PATCH 64/72] OsQuery: make policy names unique ++ revert changes to creatAgentPolicy() --- x-pack/test/osquery_cypress/runner.ts | 19 ++++------- x-pack/test/osquery_cypress/utils.ts | 48 ++++++++------------------- 2 files changed, 21 insertions(+), 46 deletions(-) diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index b1ff9331326e2..2a7b9999d6ecb 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -30,18 +30,13 @@ async function setupFleetAgent({ getService }: FtrProviderContext) { await maybeCreateDockerNetwork(log); await new FleetManager(kbnClient, log).setup(); - // FIXME:PT clean up here and above - // - // const unique = Math.random().toString(32).substring(2, 6); - // const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); - // const policyEnrollmentKeyTwo = await createAgentPolicy( - // kbnClient, - // log, - // `Osquery policy (${unique})` - // ); - - const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, 'Default policy'); - const policyEnrollmentKeyTwo = await createAgentPolicy(kbnClient, log, 'Osquery policy'); + const unique = Math.random().toString(32).substring(2, 6); + const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); + const policyEnrollmentKeyTwo = await createAgentPolicy( + kbnClient, + log, + `Osquery policy (${unique})` + ); const port = config.get('servers.fleetserver.port'); diff --git a/x-pack/test/osquery_cypress/utils.ts b/x-pack/test/osquery_cypress/utils.ts index 334f64a38d0d9..810f432d14dd9 100644 --- a/x-pack/test/osquery_cypress/utils.ts +++ b/x-pack/test/osquery_cypress/utils.ts @@ -8,19 +8,13 @@ import axios from 'axios'; import semver from 'semver'; import { map } from 'lodash'; -import { - PackagePolicy, - CreatePackagePolicyResponse, - API_VERSIONS, - AGENT_POLICY_SAVED_OBJECT_TYPE, -} from '@kbn/fleet-plugin/common'; +import { PackagePolicy, CreatePackagePolicyResponse, API_VERSIONS } from '@kbn/fleet-plugin/common'; import { KbnClient } from '@kbn/test'; import { GetEnrollmentAPIKeysResponse, CreateAgentPolicyResponse, } from '@kbn/fleet-plugin/common/types'; import { ToolingLog } from '@kbn/tooling-log'; -import { fetchAgentPolicyList } from '@kbn/security-solution-plugin/scripts/endpoint/common/fleet_services'; export const DEFAULT_HEADERS = Object.freeze({ 'x-elastic-internal-product': 'security-solution', @@ -52,33 +46,19 @@ export const createAgentPolicy = async ( data: { item: { id: agentPolicyId }, }, - } = await fetchAgentPolicyList(kbnClient, { - kuery: `${AGENT_POLICY_SAVED_OBJECT_TYPE}.name: "${agentPolicyName}"`, - }).then((response) => { - if (response.items[0]) { - log.debug(`Policy with name [${agentPolicyName}] already exists. Re-using it`); - - return { - data: { - item: response.items[0], - }, - }; - } - - return kbnClient.request({ - method: 'POST', - path: `/api/fleet/agent_policies?sys_monitoring=true`, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - body: { - name: agentPolicyName, - description: '', - namespace: 'default', - monitoring_enabled: ['logs', 'metrics'], - inactivity_timeout: 1209600, - }, - }); + } = await kbnClient.request({ + method: 'POST', + path: `/api/fleet/agent_policies?sys_monitoring=true`, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + body: { + name: agentPolicyName, + description: '', + namespace: 'default', + monitoring_enabled: ['logs', 'metrics'], + inactivity_timeout: 1209600, + }, }); log.info(`Adding integration to ${agentPolicyId}`); From 7e3c7acf85786de449694ee0fe367e83f51694cf Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 10 Nov 2023 07:34:37 -0500 Subject: [PATCH 65/72] osquery: fix failing tests --- .../plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts | 4 ++-- .../plugins/osquery/cypress/e2e/all/packs_integration.cy.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts index 64cb28d93d22c..8d9aeff21024e 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts @@ -218,8 +218,8 @@ describe('Packs - Create and Edit', { tags: ['@ess', '@serverless'] }, () => { 'Elastic-Api-Version': API_VERSIONS.internal.v1, }, }).then((response) => { - const item = response.body.items.find( - (policy: PackagePolicy) => policy.name === `Policy for ${DEFAULT_POLICY}` + const item = response.body.items.find((policy: PackagePolicy) => + policy.name.startsWith(`Policy for ${DEFAULT_POLICY}`) ); expect(item?.inputs[0].config?.osquery.value.packs[packName].queries).to.deep.equal( diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts index 73dc45837e2b3..70a5ba642d513 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts @@ -292,8 +292,8 @@ describe('ALL - Packs', { tags: ['@ess', '@serverless'] }, () => { 'Elastic-Api-Version': API_VERSIONS.internal.v1, }, }).then((response) => { - const shardPolicy = response.body.items.find( - (policy: PackagePolicy) => policy.name === `Policy for ${DEFAULT_POLICY}` + const shardPolicy = response.body.items.find((policy: PackagePolicy) => + policy.name.startsWith(`Policy for ${DEFAULT_POLICY}`) ); expect(shardPolicy?.inputs[0].config?.osquery.value.packs[shardPack]).to.deep.equal({ From a9c6861fba6ebe7bca5ea1c0db6057d2b10c0815 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 10 Nov 2023 08:18:24 -0500 Subject: [PATCH 66/72] Fix check of fleet server version when running serverless --- .../fleet_server/fleet_server_services.ts | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index d6944c74ac31b..f6590e5ef9e2e 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -330,18 +330,24 @@ const startFleetServerWithDocker = async ({ await waitForHostToEnroll(kbnClient, log, hostname, 120000); } - fleetServerVersionInfo = ( - await execa('docker', [ - 'exec', - containerName, - '/bin/bash', - '-c', - '/usr/share/elastic-agent/elastic-agent version', - ]).catch((err) => { - log.verbose(`Failed to retrieve agent version information from running instance.`, err); - return { stdout: 'Unable to retrieve version information' }; - }) - ).stdout; + fleetServerVersionInfo = isServerless + ? // `/usr/bin/fleet-server` process does not seem to support a `--version` type of argument + 'Running latest standalone fleet server' + : ( + await execa('docker', [ + 'exec', + containerName, + '/bin/bash', + '-c', + '/usr/share/elastic-agent/elastic-agent version', + ]).catch((err) => { + log.verbose( + `Failed to retrieve agent version information from running instance.`, + err + ); + return { stdout: 'Unable to retrieve version information' }; + }) + ).stdout; } catch (error) { log.error(dump(error)); throw error; From 894a32a294784392debd83f6a3ea3ae1dc6cc39a Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 10 Nov 2023 08:30:55 -0500 Subject: [PATCH 67/72] osQuery: only attempt to cleanup agent policy if we have ID --- .../plugins/osquery/cypress/e2e/all/packs_integration.cy.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts index 70a5ba642d513..01fe8139f25b7 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts @@ -203,7 +203,10 @@ describe('ALL - Packs', { tags: ['@ess', '@serverless'] }, () => { afterEach(() => { cleanupPack(globalPackId); - cleanupAgentPolicy(agentPolicyId); + + if (agentPolicyId) { + cleanupAgentPolicy(agentPolicyId); + } }); it('add global packs to policies', () => { From d5902dafc5b999355c2cfefdd07490b7fe8bfe92 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 10 Nov 2023 09:46:13 -0500 Subject: [PATCH 68/72] osquery: ensure fleet server is setup with correct port --- x-pack/test/osquery_cypress/fleet_server.ts | 7 ++++++- x-pack/test/osquery_cypress/runner.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/test/osquery_cypress/fleet_server.ts b/x-pack/test/osquery_cypress/fleet_server.ts index b3ea2277d71f3..264bf9f869891 100644 --- a/x-pack/test/osquery_cypress/fleet_server.ts +++ b/x-pack/test/osquery_cypress/fleet_server.ts @@ -17,7 +17,11 @@ import { getLatestAvailableAgentVersion } from './utils'; export class FleetManager extends Manager { private fleetServer: StartedFleetServer | undefined = undefined; - constructor(private readonly kbnClient: KbnClient, private readonly log: ToolingLog) { + constructor( + private readonly kbnClient: KbnClient, + private readonly log: ToolingLog, + private readonly port: number + ) { super(); } @@ -26,6 +30,7 @@ export class FleetManager extends Manager { this.fleetServer = await startFleetServer({ kbnClient: this.kbnClient, logger: this.log, + port: this.port, force: true, version, }); diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 2a7b9999d6ecb..ac261ad27e9d7 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -28,7 +28,7 @@ async function setupFleetAgent({ getService }: FtrProviderContext) { await verifyDockerInstalled(log); await maybeCreateDockerNetwork(log); - await new FleetManager(kbnClient, log).setup(); + await new FleetManager(kbnClient, log, config.get('servers.fleetserver.port')).setup(); const unique = Math.random().toString(32).substring(2, 6); const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); From a3da54957889c83d0c5f25e7c5349aca64ffc0b6 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 10 Nov 2023 09:49:10 -0500 Subject: [PATCH 69/72] Use fleet port number when starting fleet-server --- .../security_solution/scripts/run_cypress/parallel.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index dd2b09bdb66b1..b5ab801c3fa21 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -341,9 +341,10 @@ ${JSON.stringify( fleetServer = await startFleetServer({ kbnClient, logger: log, - port: config.has('servers.fleetserver.port') - ? (config.get('servers.fleetserver.port') as number) - : undefined, + port: + fleetServerPort ?? config.has('servers.fleetserver.port') + ? (config.get('servers.fleetserver.port') as number) + : undefined, // `force` is needed to ensure that any currently running fleet server (perhaps left // over from an interrupted run) is killed and a new one restarted force: true, From ffff6d8e5cfba5a695c7fe8f4d76189395900c57 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Fri, 10 Nov 2023 13:14:00 -0500 Subject: [PATCH 70/72] set tooling log level back to `info` --- .../public/management/cypress/cypress_base.config.ts | 2 +- x-pack/test/osquery_cypress/runner.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts index dd66df5a5da72..6ed65f031d714 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress_base.config.ts @@ -54,7 +54,7 @@ export const getCypressBaseConfig = ( // Default log level for instance of `ToolingLog` created via `crateToolingLog()`. Set this // to `debug` or `verbose` when wanting to debug tooling used by tests (ex. data indexer functions). - TOOLING_LOG_LEVEL: 'debug', + TOOLING_LOG_LEVEL: 'info', // Variable works in conjunction with the Cypress parallel runner. When set to true, fleet server // will be setup right after the Kibana stack, so that by the time cypress tests `.run()`/`.open()`, diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index ac261ad27e9d7..9662440d030a3 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -18,7 +18,7 @@ import { createAgentPolicy } from './utils'; async function setupFleetAgent({ getService }: FtrProviderContext) { // Un-comment line below to set tooling log levels to verbose. Useful when debugging - createToolingLogger.defaultLogLevel = 'verbose'; + // createToolingLogger.defaultLogLevel = 'verbose'; // const log = getService('log'); const config = getService('config'); From ae2e8f6436bcc762d0864936e52296d15f3bf649 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Sun, 12 Nov 2023 10:35:59 -0500 Subject: [PATCH 71/72] osQuery: try backout of unique policy name --- x-pack/test/osquery_cypress/runner.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 9662440d030a3..579e85517be4d 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -30,13 +30,15 @@ async function setupFleetAgent({ getService }: FtrProviderContext) { await maybeCreateDockerNetwork(log); await new FleetManager(kbnClient, log, config.get('servers.fleetserver.port')).setup(); - const unique = Math.random().toString(32).substring(2, 6); - const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); - const policyEnrollmentKeyTwo = await createAgentPolicy( - kbnClient, - log, - `Osquery policy (${unique})` - ); + // const unique = Math.random().toString(32).substring(2, 6); + // const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); + // const policyEnrollmentKeyTwo = await createAgentPolicy( + // kbnClient, + // log, + // `Osquery policy (${unique})` + // ); + const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy`); + const policyEnrollmentKeyTwo = await createAgentPolicy(kbnClient, log, `Osquery policy`); const port = config.get('servers.fleetserver.port'); From 20e7851840a96221546d0a060fefeb0ac99938b3 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Sun, 12 Nov 2023 11:56:55 -0500 Subject: [PATCH 72/72] OSQuery: Revert changes to tests --- .../osquery/cypress/e2e/all/packs_create_edit.cy.ts | 4 ++-- .../osquery/cypress/e2e/all/packs_integration.cy.ts | 9 +++------ x-pack/test/osquery_cypress/runner.ts | 7 ------- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts index 09ae67153a6fa..32e2496f8ea06 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts @@ -218,8 +218,8 @@ describe('Packs - Create and Edit', { tags: ['@ess', '@serverless'] }, () => { 'Elastic-Api-Version': API_VERSIONS.internal.v1, }, }).then((response) => { - const item = response.body.items.find((policy: PackagePolicy) => - policy.name.startsWith(`Policy for ${DEFAULT_POLICY}`) + const item = response.body.items.find( + (policy: PackagePolicy) => policy.name === `Policy for ${DEFAULT_POLICY}` ); expect(item?.inputs[0].config?.osquery.value.packs[packName].queries).to.deep.equal( diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts index e68b2924abced..30b8a1641fb94 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/packs_integration.cy.ts @@ -203,10 +203,7 @@ describe('ALL - Packs', { tags: ['@ess', '@serverless'] }, () => { afterEach(() => { cleanupPack(globalPackId); - - if (agentPolicyId) { - cleanupAgentPolicy(agentPolicyId); - } + cleanupAgentPolicy(agentPolicyId); }); it('add global packs to policies', () => { @@ -295,8 +292,8 @@ describe('ALL - Packs', { tags: ['@ess', '@serverless'] }, () => { 'Elastic-Api-Version': API_VERSIONS.internal.v1, }, }).then((response) => { - const shardPolicy = response.body.items.find((policy: PackagePolicy) => - policy.name.startsWith(`Policy for ${DEFAULT_POLICY}`) + const shardPolicy = response.body.items.find( + (policy: PackagePolicy) => policy.name === `Policy for ${DEFAULT_POLICY}` ); expect(shardPolicy?.inputs[0].config?.osquery.value.packs[shardPack]).to.deep.equal({ diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 579e85517be4d..486305a41cfc0 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -30,13 +30,6 @@ async function setupFleetAgent({ getService }: FtrProviderContext) { await maybeCreateDockerNetwork(log); await new FleetManager(kbnClient, log, config.get('servers.fleetserver.port')).setup(); - // const unique = Math.random().toString(32).substring(2, 6); - // const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy (${unique})`); - // const policyEnrollmentKeyTwo = await createAgentPolicy( - // kbnClient, - // log, - // `Osquery policy (${unique})` - // ); const policyEnrollmentKey = await createAgentPolicy(kbnClient, log, `Default policy`); const policyEnrollmentKeyTwo = await createAgentPolicy(kbnClient, log, `Osquery policy`);