[CI] Archive logs from yarn es docker runs (#189231)

## Summary
The problem we're trying to solve here is to get access to
`elasticsearch-serverless` logs when they're started in docker
containers in the background (and `elasticsearch`, although currently we
don't test against that in docker for now).
## Solution
In essence:
- we needed to remove the `--rm` flag, this would allow for the
containers to stay present after they're done.
- after this, we can run `docker logs ...` on FTR post-hooks, save
these, then archive these files to buildkite
- because the containers are not removed upon finishing, we need to
clean up dangling containers before starting up

Backporting is probably not necessary, because this is only applicable
for serverless - and serverless is only supposed to run on main.

Solves: #191505
(cherry picked from commit bce4a17)

.buildkite/scripts/lifecycle/post_command.sh

@@ -35,6 +35,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
   buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
+  buildkite-agent artifact upload '.es/es*.log'
 
   if [[ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]]; then
     if [[ $BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG == 'elasticsearch-serverless-intake' ]]; then

packages/kbn-es/src/utils/docker.test.ts

@@ -663,12 +663,13 @@ describe('runServerlessCluster()', () => {
 
     // docker version (1)
     // docker ps (1)
+    // docker container rm (3)
     // docker network create (1)
     // docker pull (1)
     // docker inspect (1)
     // docker run (3)
     // docker logs (1)
-    expect(execa.mock.calls).toHaveLength(9);
+    expect(execa.mock.calls).toHaveLength(12);
   });
 
   test(`should wait for serverless nodes to return 'green' status`, async () => {
@@ -804,11 +805,12 @@ describe('runDockerContainer()', () => {
     await expect(runDockerContainer(log, {})).resolves.toBeUndefined();
     // docker version (1)
     // docker ps (1)
+    // docker container rm (3)
     // docker network create (1)
     // docker pull (1)
     // docker inspect (1)
     // docker run (1)
-    expect(execa.mock.calls).toHaveLength(6);
+    expect(execa.mock.calls).toHaveLength(9);
   });
 });
 

packages/kbn-es/src/utils/docker.ts

@@ -110,8 +110,6 @@ const DOCKER_REGISTRY = 'docker.elastic.co';
 const DOCKER_BASE_CMD = [
   'run',
 
-  '--rm',
-
   '-t',
 
   '--net',
@@ -148,8 +146,6 @@ export const ES_SERVERLESS_DEFAULT_IMAGE = `${ES_SERVERLESS_REPO_KIBANA}:${ES_SE
 const SHARED_SERVERLESS_PARAMS = [
   'run',
 
-  '--rm',
-
   '--detach',
 
   '--interactive',
@@ -382,7 +378,6 @@ export async function maybeCreateDockerNetwork(log: ToolingLog) {
 }
 
 /**
- *
  * Pull a Docker image if needed. Ensures latest image.
  * Stops serverless from pulling the same image in each node's promise and
  * gives better control of log output, instead of falling back to docker run.
@@ -417,6 +412,24 @@ export async function printESImageInfo(log: ToolingLog, image: string) {
   log.info(`Using ES image: ${imageFullName} (${revisionUrl})`);
 }
 
+export async function cleanUpDanglingContainers(log: ToolingLog) {
+  log.info(chalk.bold('Cleaning up dangling Docker containers.'));
+
+  try {
+    const serverlessContainerNames = SERVERLESS_NODES.map(({ name }) => name);
+
+    for (const name of serverlessContainerNames) {
+      await execa('docker', ['container', 'rm', name, '--force']).catch(() => {
+        // Ignore errors if the container doesn't exist
+      });
+    }
+
+    log.success('Cleaned up dangling Docker containers.');
+  } catch (e) {
+    log.error(e);
+  }
+}
+
 export async function detectRunningNodes(
   log: ToolingLog,
   options: ServerlessOptions | DockerOptions
@@ -428,19 +441,19 @@ export async function detectRunningNodes(
   }, []);
 
   const { stdout } = await execa('docker', ['ps', '--quiet'].concat(namesCmd));
-  const runningNodes = stdout.split(/\r?\n/).filter((s) => s);
+  const runningNodeIds = stdout.split(/\r?\n/).filter((s) => s);
 
-  if (runningNodes.length) {
+  if (runningNodeIds.length) {
     if (options.kill) {
       log.info(chalk.bold('Killing running ES Nodes.'));
-      await execa('docker', ['kill'].concat(runningNodes));
-
-      return;
+      await execa('docker', ['kill'].concat(runningNodeIds));
+    } else {
+      throw createCliError(
+        'ES has already been started, pass --kill to automatically stop the nodes on startup.'
+      );
     }
-
-    throw createCliError(
-      'ES has already been started, pass --kill to automatically stop the nodes on startup.'
-    );
+  } else {
+    log.info('No running nodes detected.');
   }
 }
 
@@ -458,6 +471,7 @@ async function setupDocker({
 }) {
   await verifyDockerInstalled(log);
   await detectRunningNodes(log, options);
+  await cleanUpDanglingContainers(log);
   await maybeCreateDockerNetwork(log);
   await maybePullDockerImage(log, image);
   await printESImageInfo(log, image);
@@ -748,6 +762,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
   const volumeCmd = await setupServerlessVolumes(log, options);
   const portCmd = resolvePort(options);
 
+  // This is where nodes are started
   const nodeNames = await Promise.all(
     SERVERLESS_NODES.map(async (node, i) => {
       await runServerlessEsNode(log, {

packages/kbn-es/src/utils/extract_and_archive_logs.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { ToolingLog } from '@kbn/tooling-log';
+
+import execa from 'execa';
+import Fsp from 'fs/promises';
+import { join } from 'path';
+
+import { REPO_ROOT } from '@kbn/repo-info';
+
+/**
+ * Extracts logs from Docker nodes, writes them to files, and returns the file paths.
+ */
+export async function extractAndArchiveLogs({
+  outputFolder,
+  log,
+  nodeNames,
+}: {
+  log: ToolingLog;
+  nodeNames?: string[];
+  outputFolder?: string;
+}) {
+  outputFolder = outputFolder || join(REPO_ROOT, '.es');
+  const logFiles: string[] = [];
+
+  if (!nodeNames) {
+    const { stdout: nodeNamesString } = await execa('docker', [
+      'ps',
+      '-a',
+      '--format',
+      '{{.Names}}',
+    ]);
+    nodeNames = nodeNamesString.split('\n').filter(Boolean);
+  }
+
+  if (!nodeNames.length) {
+    log.info('No Docker nodes found to extract logs from');
+    return;
+  } else {
+    log.info(`Attempting to extract logs from Docker nodes to ${outputFolder}`);
+  }
+
+  for (const name of nodeNames) {
+    const { stdout: nodeId } = await execa('docker', [
+      'ps',
+      '-a',
+      '--quiet',
+      '--filter',
+      `name=${name}`,
+    ]);
+    if (!nodeId) {
+      continue;
+    }
+
+    const { stdout } = await execa('docker', ['logs', name]);
+    const targetFile = `${name}-${nodeId}.log`;
+    const targetPath = join(outputFolder, targetFile);
+
+    await Fsp.writeFile(targetPath, stdout);
+    logFiles.push(targetFile);
+
+    log.info(`Archived logs for ${name} to ${targetPath}`);
+  }
+
+  return logFiles;
+}

packages/kbn-es/src/utils/index.ts

@@ -19,3 +19,4 @@ export * from './parse_timeout_to_ms';
 export * from './docker';
 export * from './serverless_file_realm';
 export * from './read_roles_from_resource';
+export * from './extract_and_archive_logs';

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts

@@ -12,7 +12,7 @@ import type { ToolingLog } from '@kbn/tooling-log';
 import getPort from 'get-port';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { ArtifactLicense, ServerlessProjectType } from '@kbn/es';
-import { isServerlessProjectType } from '@kbn/es/src/utils';
+import { isServerlessProjectType, extractAndArchiveLogs } from '@kbn/es/src/utils';
 import type { Config } from '../../functional_test_runner';
 import { createTestEsCluster, esTestConfig } from '../../es';
 
@@ -90,6 +90,7 @@ export async function runElasticsearch(
     });
     return async () => {
       await node.cleanup();
+      await extractAndArchiveLogs({ outputFolder: logsDir, log });
     };
   }
 
@@ -118,6 +119,7 @@ export async function runElasticsearch(
   return async () => {
     await localNode.cleanup();
     await remoteNode.cleanup();
+    await extractAndArchiveLogs({ outputFolder: logsDir, log });
   };
 }