From ff0351eb5edd7f57ab856c48f7aa45c6ae95e502 Mon Sep 17 00:00:00 2001 From: Konrad Szwarc Date: Thu, 21 Dec 2023 15:59:41 +0100 Subject: [PATCH] [EDR Workflows][Serverless] E2E Endpoint creation fine tuning (#172463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR addresses 3 known issues with environment setup on Serverless CI pipelines. ### Setup task fails on: ### 1. Host recreation after first failed attempt Test fails on: `AssertionError: Timed out retrying after 60000ms: Expected to find content: 'test-host-2321' but never did.` Failed job from before: [here](https://buildkite.com/elastic/kibana-on-merge/builds/38728#018c219b-c0b2-41a6-9cba-2613fa85382c) Endpoint creation task is successful (host is enrolled with fleet), however, it doesn't appear in kibana. Since we are using /metadata endpoint to list all agents I've added a check in the task to see if the endpoint makes it to the metadata . If it fails to do so I delete the endpoint and do a retry with additional index (.fleet-agents, metadata-current and metadata-unified) search (thanks @joeypoon) Successful job: [here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4284#018c462c-21ec-44c0-afea-a630253e7717) ### 2. Fleet server not coming up Test fails on: `│ERROR Error: Timed out waiting for fleet server [dev-fleet-server.8284.gns5] to register with Elasticsarch` Failed job from before: [here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4166#018c368c-2434-4a2a-aaab-ae097ef26843) If first attempt at creating and enrolling fleet server fails we do a retry. Successful job: [here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4285#018c462c-487d-4a1c-8a37-71ca62915878) ### 3. Package policy creation fails Test fails on: `CypressError: cy.task('indexFleetEndpointPolicy') failed with the following error: Request failed with status code 500` **Couldn't recreate in CI.** Failed job from before: [here](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4204#018c3f17-3a3a-43dd-a31b-6bc5109d4193) Package installation fails with `no_shard_available_action_exception` error. We retry api call. closes https://github.com/elastic/kibana/issues/170482 (agent creation) closes https://github.com/elastic/kibana/issues/172920 (agent creation) closes https://github.com/elastic/kibana/issues/172319 (agent creation) closes https://github.com/elastic/kibana/issues/172326 (package policy) 1000 test runs on single test file (issues were occuring in setup tasks, not test cases itself): https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4496 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4497 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4498 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4499 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4500 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4501 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4502 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4503 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4504 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4505 --- .../index_fleet_endpoint_policy.ts | 64 ++++- .../endpoint}/format_axios_error.ts | 0 .../create_and_enroll_endpoint_host_ci.ts | 12 +- .../cypress/support/data_loaders.ts | 8 +- .../common/endpoint_metadata_services.ts | 25 +- .../fleet_server/fleet_server_services.ts | 224 +++++++++--------- .../scripts/endpoint/common/fleet_services.ts | 41 ++-- .../common/random_policy_id_generator.ts | 2 +- .../endpoint/common/role_and_user_loader.ts | 2 +- .../scripts/endpoint/common/stack_services.ts | 2 +- .../endpoint/sentinelone_host/common.ts | 2 +- 11 files changed, 226 insertions(+), 156 deletions(-) rename x-pack/plugins/security_solution/{scripts/endpoint/common => common/endpoint}/format_axios_error.ts (100%) diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_endpoint_policy.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_endpoint_policy.ts index 558d0a2fa6a50..d122daf52627b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_endpoint_policy.ts +++ b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_endpoint_policy.ts @@ -22,17 +22,23 @@ import { API_VERSIONS, } from '@kbn/fleet-plugin/common'; import { memoize } from 'lodash'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { catchAxiosErrorFormatAndThrow } from '../format_axios_error'; import { usageTracker } from './usage_tracker'; import { getEndpointPackageInfo } from '../utils/package'; import type { PolicyData } from '../types'; import { policyFactory as policyConfigFactory } from '../models/policy_config'; -import { wrapErrorAndRejectPromise } from './utils'; +import { RETRYABLE_TRANSIENT_ERRORS, retryOnError, wrapErrorAndRejectPromise } from './utils'; export interface IndexedFleetEndpointPolicyResponse { integrationPolicies: PolicyData[]; agentPolicies: AgentPolicy[]; } +enum TimeoutsInMS { + TEN_SECONDS = 10 * 1000, + FIVE_MINUTES = 5 * 60 * 1000, +} /** * Create an endpoint Integration Policy (and associated Agent Policy) via Fleet * (NOTE: ensure that fleet is setup first before calling this loading function) @@ -43,7 +49,8 @@ export const indexFleetEndpointPolicy = usageTracker.track( kbnClient: KbnClient, policyName: string, endpointPackageVersion?: string, - agentPolicyName?: string + agentPolicyName?: string, + log?: ToolingLog ): Promise => { const response: IndexedFleetEndpointPolicyResponse = { integrationPolicies: [], @@ -84,6 +91,7 @@ export const indexFleetEndpointPolicy = usageTracker.track( // Create integration (package) policy const newPackagePolicyData: CreatePackagePolicyRequest['body'] = { name: policyName, + // skip_ensure_installed: true, description: 'Protect the worlds data', policy_id: agentPolicy.data.item.id, enabled: true, @@ -106,18 +114,48 @@ export const indexFleetEndpointPolicy = usageTracker.track( version: packageVersion, }, }; - const packagePolicy = (await kbnClient - .request({ - path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN, - method: 'POST', - body: newPackagePolicyData, - headers: { - 'elastic-api-version': API_VERSIONS.public.v1, - }, - }) - .catch(wrapErrorAndRejectPromise)) as AxiosResponse; - response.integrationPolicies.push(packagePolicy.data.item as PolicyData); + const createPackagePolicy = async (): Promise => + kbnClient + .request({ + path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN, + method: 'POST', + body: newPackagePolicyData, + headers: { + 'elastic-api-version': API_VERSIONS.public.v1, + }, + }) + .catch(catchAxiosErrorFormatAndThrow) + .then((res) => res.data); + + const started = new Date(); + const hasTimedOut = (): boolean => { + const elapsedTime = Date.now() - started.getTime(); + return elapsedTime > TimeoutsInMS.FIVE_MINUTES; + }; + + let packagePolicy: CreatePackagePolicyResponse | undefined; + log?.debug(`Creating integration policy with name: ${policyName}`); + + while (!packagePolicy && !hasTimedOut()) { + packagePolicy = await retryOnError( + async () => createPackagePolicy(), + [...RETRYABLE_TRANSIENT_ERRORS, 'resource_not_found_exception'], + log + ); + + if (!packagePolicy) { + await new Promise((resolve) => setTimeout(resolve, TimeoutsInMS.TEN_SECONDS)); + } + } + + if (!packagePolicy) { + throw new Error(`Create package policy failed`); + } + + log?.verbose(`Integration policy created:`, JSON.stringify(packagePolicy, null, 2)); + + response.integrationPolicies.push(packagePolicy.item as PolicyData); return response; } diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/format_axios_error.ts b/x-pack/plugins/security_solution/common/endpoint/format_axios_error.ts similarity index 100% rename from x-pack/plugins/security_solution/scripts/endpoint/common/format_axios_error.ts rename to x-pack/plugins/security_solution/common/endpoint/format_axios_error.ts diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts b/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts index df3a5cf6d38a0..ef28ebc445e2a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts @@ -6,6 +6,8 @@ */ import { kibanaPackageJson } from '@kbn/repo-info'; +import type { Client } from '@elastic/elasticsearch'; + import type { ToolingLog } from '@kbn/tooling-log'; import type { KbnClient } from '@kbn/test/src/kbn_client'; import { isFleetServerRunning } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; @@ -28,6 +30,7 @@ import { export interface CreateAndEnrollEndpointHostCIOptions extends Pick { + esClient: Client; kbnClient: KbnClient; log: ToolingLog; /** The fleet Agent Policy ID to use for enrolling the agent */ @@ -51,6 +54,7 @@ export interface CreateAndEnrollEndpointHostCIResponse { */ export const createAndEnrollEndpointHostCI = async ({ kbnClient, + esClient, log, agentPolicyId, cpus, @@ -122,7 +126,13 @@ export const createAndEnrollEndpointHostCI = async ({ await hostVm.exec(agentEnrollCommand); - const { id: agentId } = await waitForHostToEnroll(kbnClient, log, hostVm.name, 240000); + const { id: agentId } = await waitForHostToEnroll( + kbnClient, + log, + hostVm.name, + 5 * 60 * 1000, + esClient + ); return { hostname: hostVm.name, diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 1c6e50b1cf2f4..356082f7325da 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -169,12 +169,13 @@ export const dataLoaders = ( endpointPackageVersion?: string; agentPolicyName?: string; }) => { - const { kbnClient } = await stackServicesPromise; + const { kbnClient, log } = await stackServicesPromise; return indexFleetEndpointPolicy( kbnClient, policyName, endpointPackageVersion, - agentPolicyName + agentPolicyName, + log ); }, @@ -390,7 +391,7 @@ ${s1Info.status} createEndpointHost: async ( options: Omit ): Promise => { - const { kbnClient, log } = await stackServicesPromise; + const { kbnClient, log, esClient } = await stackServicesPromise; let retryAttempt = 0; const attemptCreateEndpointHost = @@ -403,6 +404,7 @@ ${s1Info.status} ...options, log, kbnClient, + esClient, }) : await createAndEnrollEndpointHost({ useClosestVersionMatch: true, diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_metadata_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_metadata_services.ts index a69f348c366eb..cf0672b113ac7 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_metadata_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_metadata_services.ts @@ -10,7 +10,11 @@ import type { KbnClient } from '@kbn/test'; import type { WriteResponseBase } from '@elastic/elasticsearch/lib/api/types'; import { clone, merge } from 'lodash'; import type { DeepPartial } from 'utility-types'; -import { catchAxiosErrorFormatAndThrow } from './format_axios_error'; +import { + RETRYABLE_TRANSIENT_ERRORS, + retryOnError, +} from '../../../common/endpoint/data_loaders/utils'; +import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import type { GetMetadataListRequestQuery } from '../../../common/api/endpoint'; import { resolvePathVariables } from '../../../public/common/utils/resolve_path_variables'; import { @@ -19,6 +23,7 @@ import { METADATA_DATASTREAM, } from '../../../common/endpoint/constants'; import type { HostInfo, HostMetadata, MetadataListResponse } from '../../../common/endpoint/types'; +import { HostStatus } from '../../../common/endpoint/types'; import { EndpointDocGenerator } from '../../../common/endpoint/generate_data'; const endpointGenerator = new EndpointDocGenerator(); @@ -163,15 +168,15 @@ export const waitForEndpointToStreamData = async ( let found: HostInfo | undefined; while (!found && !hasTimedOut()) { - found = await fetchEndpointMetadata(kbnClient, endpointAgentId).catch((error) => { - // Ignore `not found` (404) responses. Endpoint could be new and thus documents might not have - // been streamed yet. - if (error?.response?.status === 404) { - return undefined; - } - - throw error; - }); + found = await retryOnError( + async () => + fetchEndpointMetadataList(kbnClient, { + kuery: `united.endpoint.agent.id: "${endpointAgentId}"`, + }).then((response) => { + return response.data.filter((record) => record.host_status === HostStatus.HEALTHY)[0]; + }), + RETRYABLE_TRANSIENT_ERRORS + ); if (!found) { // sleep and check again diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts index bb322482feebf..c9d5b84eb167b 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts @@ -49,8 +49,8 @@ import { retryOnError, } from '../../../../common/endpoint/data_loaders/utils'; import { isServerlessKibanaFlavor } from '../stack_services'; -import type { FormattedAxiosError } from '../format_axios_error'; -import { catchAxiosErrorFormatAndThrow } from '../format_axios_error'; +import type { FormattedAxiosError } from '../../../../common/endpoint/format_axios_error'; +import { catchAxiosErrorFormatAndThrow } from '../../../../common/endpoint/format_axios_error'; import { ensureFleetSetup, fetchFleetOutputs, @@ -247,113 +247,120 @@ const startFleetServerWithDocker = async ({ let agentVersion = version || (await getAgentVersionMatchingCurrentStack(kbnClient)); const localhostRealIp = getLocalhostRealIp(); const fleetServerUrl = `https://${localhostRealIp}:${port}`; + const isServerless = await isServerlessKibanaFlavor(kbnClient); + const esURL = new URL(await getFleetElasticsearchOutputHost(kbnClient)); + const containerName = `dev-fleet-server.${port}`; log.info( `Starting a new fleet server using Docker\n Agent version: ${agentVersion}\n Server URL: ${fleetServerUrl}` ); - const response: StartedServer = await log.indent(4, async () => { - const isServerless = await isServerlessKibanaFlavor(kbnClient); - const esURL = new URL(await getFleetElasticsearchOutputHost(kbnClient)); - const containerName = `dev-fleet-server.${port}`; - const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`; - let containerId = ''; - let fleetServerVersionInfo = ''; - - if (isLocalhost(esURL.hostname)) { - esURL.hostname = localhostRealIp; - } + let retryAttempt = isServerless ? 0 : 1; + const attemptServerlessFleetServerSetup = async (): Promise => { + return log.indent(4, async () => { + const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`; + let containerId = ''; + let fleetServerVersionInfo = ''; + + if (isLocalhost(esURL.hostname)) { + esURL.hostname = localhostRealIp; + } - if (isServerless) { - log.info(`Kibana running in serverless mode. + if (isServerless) { + log.info(`Kibana running in serverless mode. - will install/run standalone Fleet Server - version adjusted to [latest] from [${agentVersion}]`); - agentVersion = 'latest'; - } else { - assert.ok(!!policyId, '`policyId` is required'); - assert.ok(!!serviceToken, '`serviceToken` is required'); - } - - // Create the `elastic` network to use with all containers - await maybeCreateDockerNetwork(log); + agentVersion = 'latest'; + } else { + assert.ok(!!policyId, '`policyId` is required'); + assert.ok(!!serviceToken, '`serviceToken` is required'); + } - try { - const dockerArgs = isServerless - ? getFleetServerStandAloneDockerArgs({ - containerName, - hostname, - port, - esUrl: esURL.toString(), - agentVersion, + // Create the `elastic` network to use with all containers + await maybeCreateDockerNetwork(log); + try { + const dockerArgs = isServerless + ? getFleetServerStandAloneDockerArgs({ + containerName, + hostname, + port, + esUrl: esURL.toString(), + agentVersion, + }) + : getFleetServerManagedDockerArgs({ + containerName, + hostname, + port, + serviceToken, + policyId, + agentVersion, + esUrl: esURL.toString(), + }); + + await execa('docker', ['kill', containerName]) + .then(() => { + log.info( + `Killed an existing container with name [${containerName}]. New one will be started.` + ); }) - : getFleetServerManagedDockerArgs({ - containerName, - hostname, - port, - serviceToken, - policyId, - agentVersion, - esUrl: esURL.toString(), + .catch((error) => { + if (!/no such container/i.test(error.message)) { + log.verbose(`Attempt to kill currently running fleet-server container with name [${containerName}] was unsuccessful: + ${error}`); + } }); - await execa('docker', ['kill', containerName]) - .then(() => { - log.info( - `Killed an existing container with name [${containerName}]. New one will be started.` - ); - }) - .catch((error) => { - if (!/no such container/i.test(error.message)) { - log.verbose(`Attempt to kill currently running fleet-server container with name [${containerName}] was unsuccessful: - ${error}`); - } - }); + log.verbose(`docker arguments:\n${dockerArgs.join(' ')}`); - log.verbose(`docker arguments:\n${dockerArgs.join(' ')}`); + containerId = (await execa('docker', dockerArgs)).stdout; - containerId = (await execa('docker', dockerArgs)).stdout; + log.info(`Fleet server started`); - log.info(`Fleet server started`); + if (!isServerless) { + await addFleetServerHostToFleetSettings(kbnClient, log, fleetServerUrl); + } - if (!isServerless) { - await addFleetServerHostToFleetSettings(kbnClient, log, fleetServerUrl); - } + await updateFleetElasticsearchOutputHostNames(kbnClient, log); - await updateFleetElasticsearchOutputHostNames(kbnClient, log); + if (isServerless) { + log.info(`Waiting for server [${hostname}] to register with Elasticsearch`); + await waitForFleetServerToRegisterWithElasticsearch(kbnClient, hostname, 180000); + } else { + await waitForHostToEnroll(kbnClient, log, hostname, 120000); + } - if (isServerless) { - log.info(`Waiting for server [${hostname}] to register with Elasticsearch`); + fleetServerVersionInfo = isServerless + ? // `/usr/bin/fleet-server` process does not seem to support a `--version` type of argument + 'Running latest standalone fleet server' + : ( + await execa('docker', [ + 'exec', + containerName, + '/bin/bash', + '-c', + '/usr/share/elastic-agent/elastic-agent version', + ]).catch((err) => { + log.verbose( + `Failed to retrieve agent version information from running instance.`, + err + ); + return { stdout: 'Unable to retrieve version information' }; + }) + ).stdout; + } catch (error) { + if (retryAttempt < 1) { + retryAttempt++; + log.error(`Failed to start fleet server, retrying. Error: ${error.message}`); + log.verbose(dump(error)); + return attemptServerlessFleetServerSetup(); + } - await waitForFleetServerToRegisterWithElasticsearch(kbnClient, hostname, 180000); - } else { - await waitForHostToEnroll(kbnClient, log, hostname, 120000); + log.error(dump(error)); + throw error; } - fleetServerVersionInfo = isServerless - ? // `/usr/bin/fleet-server` process does not seem to support a `--version` type of argument - 'Running latest standalone fleet server' - : ( - await execa('docker', [ - 'exec', - containerName, - '/bin/bash', - '-c', - '/usr/share/elastic-agent/elastic-agent version', - ]).catch((err) => { - log.verbose( - `Failed to retrieve agent version information from running instance.`, - err - ); - return { stdout: 'Unable to retrieve version information' }; - }) - ).stdout; - } catch (error) { - log.error(dump(error)); - throw error; - } - - const info = `Container Name: ${containerName} + const info = `Container Name: ${containerName} Container Id: ${containerId} Fleet-server version: ${fleetServerVersionInfo.replace(/\n/g, '\n ')} @@ -363,26 +370,29 @@ Shell access: ${chalk.cyan(`docker exec -it ${containerName} /bin/bash`) Kill container: ${chalk.cyan(`docker kill ${containerId}`)} `; - return { - type: 'docker', - name: containerName, - id: containerId, - url: fleetServerUrl, - info, - stop: async () => { - log.info( - `Stopping (kill) fleet server. Container name [${containerName}] id [${containerId}]` - ); - await execa('docker', ['kill', containerId]); - }, - stopNow: () => { - log.info( - `Stopping (kill) fleet server. Container name [${containerName}] id [${containerId}]` - ); - execa.sync('docker', ['kill', containerId]); - }, - }; - }); + return { + type: 'docker', + name: containerName, + id: containerId, + url: fleetServerUrl, + info, + stop: async () => { + log.info( + `Stopping (kill) fleet server. Container name [${containerName}] id [${containerId}]` + ); + await execa('docker', ['kill', containerId]); + }, + stopNow: () => { + log.info( + `Stopping (kill) fleet server. Container name [${containerName}] id [${containerId}]` + ); + execa.sync('docker', ['kill', containerId]); + }, + }; + }); + }; + + const response: StartedServer = await attemptServerlessFleetServerSetup(); log.info(`Done. Fleet server up and running`); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 1ca0e3ef12273..97a9df146c2ac 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -9,20 +9,20 @@ import { map, memoize, pick } from 'lodash'; import type { Client, estypes } from '@elastic/elasticsearch'; import type { Agent, + AgentPolicy, AgentStatus, + CreateAgentPolicyRequest, + CreateAgentPolicyResponse, + CreatePackagePolicyRequest, + CreatePackagePolicyResponse, GetAgentPoliciesRequest, GetAgentPoliciesResponse, GetAgentsResponse, + GetInfoResponse, + GetOneAgentPolicyResponse, GetPackagePoliciesRequest, GetPackagePoliciesResponse, - CreateAgentPolicyRequest, - AgentPolicy, - CreateAgentPolicyResponse, - CreatePackagePolicyResponse, - CreatePackagePolicyRequest, PackagePolicy, - GetInfoResponse, - GetOneAgentPolicyResponse, PostFleetSetupResponse, } from '@kbn/fleet-plugin/common'; import { @@ -47,13 +47,13 @@ import { outputRoutesService, } from '@kbn/fleet-plugin/common/services'; import type { + DeleteAgentPolicyResponse, EnrollmentAPIKey, + GenerateServiceTokenResponse, GetAgentsRequest, GetEnrollmentAPIKeysResponse, - PostAgentUnenrollResponse, - GenerateServiceTokenResponse, GetOutputsResponse, - DeleteAgentPolicyResponse, + PostAgentUnenrollResponse, } from '@kbn/fleet-plugin/common/types'; import nodeFetch from 'node-fetch'; import semver from 'semver'; @@ -71,7 +71,7 @@ import { wrapErrorAndRejectPromise, } from '../../../common/endpoint/data_loaders/utils'; import { fetchKibanaStatus } from './stack_services'; -import { catchAxiosErrorFormatAndThrow } from './format_axios_error'; +import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fleet_agent_generator'; const fleetGenerator = new FleetAgentGenerator(); @@ -161,12 +161,14 @@ export const fetchFleetAgents = async ( * @param log * @param hostname * @param timeoutMs + * @param esClient */ export const waitForHostToEnroll = async ( kbnClient: KbnClient, log: ToolingLog, hostname: string, - timeoutMs: number = 30000 + timeoutMs: number = 30000, + esClient: Client | undefined = undefined ): Promise => { log.info(`Waiting for host [${hostname}] to enroll with fleet`); @@ -212,6 +214,12 @@ export const waitForHostToEnroll = async ( log.debug(`Host [${hostname}] has been enrolled with fleet`); log.verbose(found); + // Workaround for united metadata sometimes being unable to find docs in .fleet-agents index. This + // seems to be a timing issue with the index refresh. + await esClient?.search({ + index: AGENTS_INDEX, + }); + return found; }; @@ -401,9 +409,7 @@ export const getAgentFileName = (agentVersion: string): string => { const downloadArch = { arm64: 'arm64', x64: 'x86_64' }[process.arch as string] ?? `UNSUPPORTED_ARCHITECTURE_${process.arch}`; - const fileName = `elastic-agent-${agentVersion}-linux-${downloadArch}`; - - return fileName; + return `elastic-agent-${agentVersion}-linux-${downloadArch}`; }; interface ElasticArtifactSearchResponse { @@ -560,8 +566,7 @@ export const unEnrollFleetAgent = async ( * Un-enrolls a Fleet agent * * @param kbnClient - * @param agentId - * @param force + * @param policyId */ export const getAgentPolicyEnrollmentKey = async ( kbnClient: KbnClient, @@ -655,7 +660,7 @@ interface EnrollHostVmWithFleetOptions { /** * Installs the Elastic agent on the provided Host VM and enrolls with it Fleet. * - * NOTE: this method assumes that FLeet-Server is already setup and running. + * NOTE: this method assumes that Fleet-Server is already setup and running. * * @param hostVm * @param kbnClient diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/random_policy_id_generator.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/random_policy_id_generator.ts index 3b494d3bfe9cb..3c58f44b3d266 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/random_policy_id_generator.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/random_policy_id_generator.ts @@ -12,7 +12,7 @@ import { PACKAGE_POLICY_API_ROUTES, PACKAGE_POLICY_SAVED_OBJECT_TYPE, } from '@kbn/fleet-plugin/common/constants'; -import { catchAxiosErrorFormatAndThrow } from './format_axios_error'; +import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import { indexFleetEndpointPolicy } from '../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; import { setupFleetForEndpoint } from '../../../common/endpoint/data_loaders/setup_fleet_for_endpoint'; import type { GetPolicyListResponse } from '../../../public/management/pages/policy/types'; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts index f8c51d5255018..eb493e22dafaf 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts @@ -14,7 +14,7 @@ import { inspect } from 'util'; import type { AxiosError } from 'axios'; import type { EndpointSecurityRoleDefinitions } from './roles_users'; import { getAllEndpointSecurityRoles } from './roles_users'; -import { catchAxiosErrorFormatAndThrow } from './format_axios_error'; +import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import { COMMON_API_HEADERS } from './constants'; const ignoreHttp409Error = (error: AxiosError) => { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts index 13fc2a1a0ffba..f95ee808408ed 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts @@ -17,7 +17,7 @@ import type { ClientOptions } from '@elastic/elasticsearch/lib/client'; import fs from 'fs'; import { CA_CERT_PATH } from '@kbn/dev-utils'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; -import { catchAxiosErrorFormatAndThrow } from './format_axios_error'; +import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import { isLocalhost } from './is_localhost'; import { getLocalhostRealIp } from './network_services'; import { createSecuritySuperuser } from './security_user_services'; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/common.ts b/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/common.ts index f15c08ee87500..483bae1c4275e 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/common.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/sentinelone_host/common.ts @@ -14,7 +14,7 @@ import type { S1AgentPackage, S1AgentPackageListApiResponse, } from './types'; -import { catchAxiosErrorFormatAndThrow } from '../common/format_axios_error'; +import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import type { HostVm } from '../common/types'; interface S1ClientOptions {