[Security Solution][Endpoint] Cypress test improvements to capture Ag…

…ent diagnostics file when test fails (elastic#202965)

## Summary

- the Cypress `parallel` runner was updated to set tooling logging level
first from Env. variables before falling back to the value defined in
the Cypress configuration file
- The env. value to set, if wanting to enable a specific logging level,
is `TOOLING_LOG_LEVEL`. The values supported are the same as those used
with `ToolingLog`
([here](https://github.com/elastic/kibana/blob/b6287708f687d4e3288851052c0c6ae4ade8ce60/packages/kbn-tooling-log/src/log_levels.ts#L10)):
`silent`, `error`, `warning`, `success`, `info`, `debug`, `verbose`
- This change makes it easier to run Cypress tests locally with (for
example) a logging level of `verbose` for our tooling without having to
modify the Cypress configuration file. Example: `export
TOOLING_LOG_LEVEL=verbose && yarn cypress:dw:open`
- Added two new methods to our scripting VM service clients (for Vagrant
and Multipass):
- `download`: allow you to pull files out of the VM and save them
locally
- `upload`: uploads a local file to the VM. (upload already existed as
`transfer` - which has now been marked as deprecated).
- Added new service function on our Fleet scripting module to enable us
to set the logging level on a Fleet Agent
- Cypress tests were adjusted to automatically set the agent logging to
debug when running in CI
- A new Cypress task that allows for an Agent Diagnostic file (which
includes the Endpoint Log) to be retrieved from the host VM and stored
with the CI job (under the artifacts tab)
    - A few tests were updated to include this step for failed test

x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts

@@ -11,6 +11,7 @@
 
 import type { CasePostRequest } from '@kbn/cases-plugin/common/api';
 import type { UsageRecord } from '@kbn/security-solution-serverless/server/types';
+import type { HostVmTransferResponse } from '../../../scripts/endpoint/common/types';
 import type {
   DeletedEndpointHeartbeats,
   IndexedEndpointHeartbeats,
@@ -30,6 +31,7 @@ import type {
   UninstallAgentFromHostTaskOptions,
   IsAgentAndEndpointUninstalledFromHostTaskOptions,
   LogItTaskOptions,
+  CaptureHostVmAgentDiagnosticsOptions,
 } from './types';
 import type {
   DeleteIndexedFleetEndpointPoliciesResponse,
@@ -267,6 +269,12 @@ declare global {
         arg: LogItTaskOptions,
         options?: Partial<Loggable & Timeoutable>
       ): Chainable<null>;
+
+      task(
+        name: 'captureHostVmAgentDiagnostics',
+        arg: CaptureHostVmAgentDiagnosticsOptions,
+        options?: Partial<Loggable & Timeoutable>
+      ): Chainable<Omit<HostVmTransferResponse, 'delete'>>;
     }
   }
 }

x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts

@@ -67,6 +67,15 @@ describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => {
       }
     });
 
+    afterEach(function () {
+      if (Cypress.env('IS_CI') && this.currentTest?.isFailed() && createdHost) {
+        cy.task('captureHostVmAgentDiagnostics', {
+          hostname: createdHost.hostname,
+          fileNamePrefix: this.currentTest?.fullTitle(),
+        });
+      }
+    });
+
     it('"get-file --path" - should retrieve a file', () => {
       const downloadsFolder = Cypress.config('downloadsFolder');
 

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -11,6 +11,10 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common';
 import execa from 'execa';
 import type { KbnClient } from '@kbn/test';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/repo-info';
+// This is a Cypress module and only used by Cypress, so disabling "should" be safe
+// eslint-disable-next-line import/no-nodejs-modules
+import { mkdir } from 'node:fs/promises';
 import type { IndexedEndpointHeartbeats } from '../../../../common/endpoint/data_loaders/index_endpoint_hearbeats';
 import {
   deleteIndexedEndpointHeartbeats,
@@ -53,6 +57,7 @@ import type {
   LoadUserAndRoleCyTaskOptions,
   CreateUserAndRoleCyTaskOptions,
   LogItTaskOptions,
+  CaptureHostVmAgentDiagnosticsOptions,
 } from '../types';
 import type {
   DeletedIndexedEndpointRuleAlerts,
@@ -75,6 +80,7 @@ import {
   deleteAgentPolicy,
   fetchAgentPolicyEnrollmentKey,
   getOrCreateDefaultAgentPolicy,
+  setAgentLoggingLevel,
 } from '../../../../scripts/endpoint/common/fleet_services';
 import { startElasticAgentWithDocker } from '../../../../scripts/endpoint/common/elastic_agent_service';
 import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
@@ -433,6 +439,7 @@ ${s1Info.status}
                   log,
                   kbnClient,
                 });
+            await setAgentLoggingLevel(kbnClient, newHost.agentId, 'debug', log);
             await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000);
             return newHost;
           } catch (err) {
@@ -531,5 +538,65 @@ ${s1Info.status}
       await startEndpointHost(hostName);
       return null;
     },
+
+    /**
+     * Generates an Agent Diagnostics archive (ZIP) directly on the Host VM and saves it to a directory
+     * that is then included with the list of Artifacts that are captured with Buildkite job.
+     *
+     * ### Usage:
+     *
+     * This task is best used from a `afterEach()` by checking if the test failed and if so (and it
+     * was a test that was running against a host VM), then capture the diagnostics file
+     *
+     * @param hostname
+     *
+     * @example
+     *
+     * describe('something', () => {
+     *   let hostVm;
+     *
+     *   afterEach(function() { // << Important: Note the use of `function()` here instead of arrow function
+     *     if (this.currentTest?.isFailed() && hostVm) {
+     *       cy.task('captureHostVmAgentDiagnostics', { hostname: hostVm.hostname });
+     *     }
+     *   });
+     *
+     *   //...
+     * })
+     */
+    captureHostVmAgentDiagnostics: async ({
+      hostname,
+      fileNamePrefix = '',
+    }: CaptureHostVmAgentDiagnosticsOptions) => {
+      const { log } = await stackServicesPromise;
+
+      log.info(`Capturing agent diagnostics for host VM [${hostname}]`);
+
+      const vmClient = getHostVmClient(hostname, undefined, undefined, log);
+      const fileName = `elastic-agent-diagnostics-${hostname}-${new Date()
+        .toISOString()
+        .replace(/:/g, '.')}.zip`;
+      const vmDiagnosticsFile = `/tmp/${fileName}`;
+      const localDiagnosticsDir = `${REPO_ROOT}/target/test_failures`;
+      const localDiagnosticsFile = `${localDiagnosticsDir}/${
+        fileNamePrefix
+          ? // Insure the file name prefix does not have characters that can't be used in file names
+            `${fileNamePrefix.replace(/[><:"/\\|?*'`{} ]/g, '_')}-`
+          : ''
+      }${fileName}`;
+
+      await mkdir(localDiagnosticsDir, { recursive: true });
+
+      // generate diagnostics file on the host and then download it
+      await vmClient.exec(
+        `sudo /opt/Elastic/Agent/elastic-agent diagnostics --file ${vmDiagnosticsFile}`
+      );
+      return vmClient.download(vmDiagnosticsFile, localDiagnosticsFile).then((response) => {
+        log.info(`Agent diagnostic file for host [${hostname}] has been downloaded and is available at:
+  ${response.filePath}
+`);
+        return { filePath: response.filePath };
+      });
+    },
   });
 };

x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts

@@ -16,10 +16,20 @@ export const setupToolingLogLevel = (config: Cypress.PluginConfigOptions) => {
   const log = createToolingLogger();
   const defaultToolingLogLevel = config.env.TOOLING_LOG_LEVEL;
 
-  log.info(`Cypress config 'env.TOOLING_LOG_LEVEL': ${defaultToolingLogLevel}`);
+  log.info(`
+
+Cypress Configuration File: ${config.configFile}
+
+'env.TOOLING_LOG_LEVEL' set to: ${defaultToolingLogLevel}
+
+*** FYI: ***  To help with test failures, an environmental variable named 'TOOLING_LOG_LEVEL' can be set
+              with a value of 'verbose' in order to capture more data in the logs. This environment
+              property can be set either in the runtime environment (ex. local shell or buildkite) or
+              directly in the Cypress configuration file \`env: {}\` section.
+
+  `);
 
   if (defaultToolingLogLevel && defaultToolingLogLevel !== createToolingLogger.defaultLogLevel) {
     createToolingLogger.defaultLogLevel = defaultToolingLogLevel;
-    log.info(`Default log level for 'createToolingLogger()' set to ${defaultToolingLogLevel}`);
   }
 };

x-pack/plugins/security_solution/public/management/cypress/types.ts

@@ -81,3 +81,8 @@ export interface LogItTaskOptions {
   level: keyof Pick<ToolingLog, 'info' | 'debug' | 'verbose'>;
   data: any;
 }
+
+export interface CaptureHostVmAgentDiagnosticsOptions {
+  hostname: string;
+  fileNamePrefix?: string;
+}

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts

@@ -277,17 +277,19 @@ const startFleetServerWithDocker = async ({
   const isServerless = await isServerlessKibanaFlavor(kbnClient);
   const esURL = new URL(await getFleetElasticsearchOutputHost(kbnClient));
   const containerName = `dev-fleet-server.${port}`;
+  let fleetServerVersionInfo = '';
 
   log.info(
     `Starting a new fleet server using Docker\n    Agent version: ${agentVersion}\n    Server URL: ${fleetServerUrl}`
   );
 
   let retryAttempt = isServerless ? 0 : 1;
   const attemptServerlessFleetServerSetup = async (): Promise<StartedServer> => {
+    fleetServerVersionInfo = '';
+
     return log.indent(4, async () => {
       const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`;
       let containerId = '';
-      let fleetServerVersionInfo = '';
 
       if (isLocalhost(esURL.hostname)) {
         esURL.hostname = localhostRealIp;
@@ -361,8 +363,17 @@ const startFleetServerWithDocker = async ({
         }
 
         fleetServerVersionInfo = isServerless
-          ? // `/usr/bin/fleet-server` process does not seem to support a `--version` type of argument
-            'Running latest standalone fleet server'
+          ? (
+              await execa
+                .command(`docker exec ${containerName} /usr/bin/fleet-server --version`)
+                .catch((err) => {
+                  log.verbose(
+                    `Failed to retrieve fleet-server (serverless/standalone) version information from running instance.`,
+                    err
+                  );
+                  return { stdout: 'Unable to retrieve version information (serverless)' };
+                })
+            ).stdout
           : (
               await execa('docker', [
                 'exec',
@@ -424,7 +435,7 @@ Kill container:       ${chalk.cyan(`docker kill ${containerId}`)}
 
   const response: StartedServer = await attemptServerlessFleetServerSetup();
 
-  log.info(`Done. Fleet server up and running`);
+  log.info(`Done. Fleet server up and running (version: ${fleetServerVersionInfo})`);
 
   return response;
 };

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts

@@ -56,12 +56,14 @@ import type {
   DeleteAgentPolicyResponse,
   EnrollmentAPIKey,
   GenerateServiceTokenResponse,
+  GetActionStatusResponse,
   GetAgentsRequest,
   GetEnrollmentAPIKeysResponse,
   GetOutputsResponse,
   PostAgentUnenrollResponse,
   UpdateAgentPolicyRequest,
   UpdateAgentPolicyResponse,
+  PostNewAgentActionResponse,
 } from '@kbn/fleet-plugin/common/types';
 import semver from 'semver';
 import axios from 'axios';
@@ -1499,3 +1501,92 @@ export const updateAgentPolicy = async (
     .catch(catchAxiosErrorFormatAndThrow)
     .then((response) => response.data.item);
 };
+
+/**
+ * Sets the log level on a Fleet agent and waits a bit of time to allow it for to
+ * complete (but does not error if it does not complete)
+ *
+ * @param kbnClient
+ * @param agentId
+ * @param logLevel
+ * @param log
+ */
+export const setAgentLoggingLevel = async (
+  kbnClient: KbnClient,
+  agentId: string,
+  logLevel: 'debug' | 'info' | 'warning' | 'error',
+  log: ToolingLog = createToolingLogger()
+): Promise<PostNewAgentActionResponse> => {
+  log.debug(`Setting fleet agent [${agentId}] logging level to [${logLevel}]`);
+
+  const response = await kbnClient
+    .request<PostNewAgentActionResponse>({
+      method: 'POST',
+      path: `/api/fleet/agents/${agentId}/actions`,
+      body: { action: { type: 'SETTINGS', data: { log_level: logLevel } } },
+      headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 },
+    })
+    .then((res) => res.data);
+
+  // Wait to see if the action completes, but don't `throw` if it does not
+  await waitForFleetAgentActionToComplete(kbnClient, response.item.id)
+    .then(() => {
+      log.debug(`Fleet action to set agent [${agentId}] logging level to [${logLevel}] completed!`);
+    })
+    .catch((err) => {
+      log.debug(err.message);
+    });
+
+  return response;
+};
+
+/**
+ * Retrieve fleet agent action statuses
+ * @param kbnClient
+ */
+export const fetchFleetAgentActionStatus = async (
+  kbnClient: KbnClient
+): Promise<GetActionStatusResponse> => {
+  return kbnClient
+    .request<GetActionStatusResponse>({
+      method: 'GET',
+      path: agentRouteService.getActionStatusPath(),
+      query: { perPage: 1000 },
+      headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 },
+    })
+    .then((response) => response.data);
+};
+
+/**
+ * Check and wait until a Fleet Agent action is complete.
+ * @param kbnClient
+ * @param actionId
+ * @param timeout
+ *
+ * @throws
+ */
+export const waitForFleetAgentActionToComplete = async (
+  kbnClient: KbnClient,
+  actionId: string,
+  timeout: number = 20_000
+): Promise<void> => {
+  await pRetry(
+    async (attempts) => {
+      const { items: actionList } = await fetchFleetAgentActionStatus(kbnClient);
+      const actionInfo = actionList.find((action) => action.actionId === actionId);
+
+      if (!actionInfo) {
+        throw new Error(
+          `Fleet Agent action id [${actionId}] was not found in list of actions retrieved from fleet!`
+        );
+      }
+
+      if (actionInfo.status === 'IN_PROGRESS') {
+        throw new Error(
+          `Fleet agent action id [${actionId}] remains in progress after [${attempts}] attempts to check its status`
+        );
+      }
+    },
+    { maxTimeout: 2_000, maxRetryTime: timeout }
+  );
+};

x-pack/plugins/security_solution/scripts/endpoint/common/types.ts

@@ -14,8 +14,12 @@ export interface HostVm {
   exec: (command: string) => Promise<HostVmExecResponse>;
   mount: (localDir: string, hostVmDir: string) => Promise<HostVmMountResponse>;
   unmount: (hostVmDir: string) => Promise<void>;
-  /** Uploads/copies a file from the local machine to the VM */
+  /** @deprecated use `upload` */
   transfer: (localFilePath: string, destFilePath: string) => Promise<HostVmTransferResponse>;
+  /** Uploads/copies a file from the local machine to the VM */
+  upload: (localFilePath: string, destFilePath: string) => Promise<HostVmTransferResponse>;
+  /** Downloads a file from the host VM to the local machine */
+  download: (vmFilePath: string, localFilePath: string) => Promise<HostVmTransferResponse>;
   destroy: () => Promise<void>;
   info: () => string;
   stop: () => void;
@@ -33,8 +37,8 @@ export interface HostVmMountResponse {
   unmount: () => Promise<void>;
 }
 export interface HostVmTransferResponse {
-  /** The file path of the file on the host vm */
+  /** The path of the file that was either uploaded to the host VM or downloaded to the local machine from the VM */
   filePath: string;
-  /** Delete the file from the host VM */
+  /** Delete the file from the host VM or from the local machine depending on what client method was used */
   delete: () => Promise<HostVmExecResponse>;
 }