From 91cdbe2d354100683b5d8670de88e0b2cf665ba9 Mon Sep 17 00:00:00 2001 From: Konrad Szwarc Date: Wed, 11 Oct 2023 16:26:45 +0200 Subject: [PATCH] [EDR Workflows][E2E] Increase the timeout of agent check in (#168438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request extends the agent fleet check timeout from 2 minutes to 4 minutes. We've identified a number of unreliable tests that fail during the `beforeAll` stage while executing the `createEndpointHost` task. The following logs appear before the timeout: ``` info Enrolling Elastic Agent with Fleet | Installing service....... DONE | Starting service... DONE | Enrolling Elastic Agent with Fleet..........Successfully enrolled the Elastic Agent. | Elastic Agent has been successfully installed. | info Waiting for Agent to check-in with Fleet ``` The error message we encounter is `> Timed out waiting for host [test-host-4981] to appear in Fleet.` It appears that all the preceding steps are successful, and only the final one fails due to either the agent not checking in with the fleet for 2 minutes or the agent being unhealthy for two minutes. Since I haven't been able to replicate this behavior locally, and there isn't a way to inspect what's happening on the agent, I believe the best course of action at this point is to extend the timeout and monitor the results. Reports of this error: closes https://github.com/elastic/kibana/issues/168427 closes https://github.com/elastic/kibana/issues/168394 closes https://github.com/elastic/kibana/issues/168393 closes https://github.com/elastic/kibana/issues/168390 closes https://github.com/elastic/kibana/issues/168363 closes https://github.com/elastic/kibana/issues/168362 closes https://github.com/elastic/kibana/issues/168361 closes https://github.com/elastic/kibana/issues/168360 closes https://github.com/elastic/kibana/issues/168359 Affected CI runs: https://buildkite.com/elastic/kibana-on-merge/builds/36483 https://buildkite.com/elastic/kibana-on-merge/builds/36497 https://buildkite.com/elastic/kibana-on-merge/builds/36501 https://buildkite.com/elastic/kibana-on-merge/builds/36526 Another time out happens from time to time when previously set 10 minutes timeout on `createEndpointHost` task is not enough to set up the environment. Its portrayed below, timeout happens during agent setup ```   | default: Running: inline script   | default: Reading package lists...   | default: Building dependency tree...   | default: Reading state information...   | default: Suggested packages:   | default: zip   | default: The following NEW packages will be installed:   | default: unzip   | default: 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.   | default: Need to get 174 kB of archives.   | default: After this operation, 385 kB of additional disk space will be used.   | default: Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.1 [174 kB]   | default: dpkg-preconfigure: unable to re-open stdin: No such file or directory   | default: Fetched 174 kB in 1s (210 kB/s)   | default: Selecting previously unselected package unzip.   | (Reading database ... 63961 files and directories currently installed.)   | default: Preparing to unpack .../unzip_6.0-26ubuntu3.1_amd64.deb ...   | default: Unpacking unzip (6.0-26ubuntu3.1) ...   | default: Setting up unzip (6.0-26ubuntu3.1) ...   | default: Processing triggers for man-db (2.10.2-1) ...   |     | CypressError: `cy.task('createEndpointHost')` timed out after waiting `600000ms`. ``` --- .../public/management/cypress/tasks/create_endpoint_host.ts | 2 +- .../scripts/endpoint/common/endpoint_host_services.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts index cac8c284acdbd..7db2ad37317fd 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts @@ -17,6 +17,6 @@ export const createEndpointHost = ( { agentPolicyId, }, - { timeout: timeout ?? 600000 } + { timeout: timeout ?? 900000 } // 15 minutes, since setup can take 10 minutes and more. Task will time out if is not resolved within this time. ); }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index 8c4bdac30362e..0be12de4ade91 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -335,7 +335,7 @@ const enrollHostWithFleet = async ({ ]); } log.info(`Waiting for Agent to check-in with Fleet`); - const agent = await waitForHostToEnroll(kbnClient, vmName, 120000); + const agent = await waitForHostToEnroll(kbnClient, vmName, 240000); return { agentId: agent.id,