From 3cf8077a4404750acfa005b5a15291a6ab11556b Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 14 Feb 2024 17:07:27 +0100 Subject: [PATCH] [kbn-journey] improve CI output by saving server logs into files (#175293) ## Summary This PR refactors `run_performance` script logic that we use to run single user performance journeys: - simplify script by keeping only 2 functions: `startES` & `runFunctionalTest` - write ES and Kibana logs (for warmup phase) to files instead of console output; - upload files with server log as build artifacts - do not retry journey TEST phase run on failure, it doubles EBT metrics and does not provide much value: journey should be simple and stable. - do not retry step running journeys, most of the failures are due to journey flakiness and we just skip them Testing performance pipeline https://buildkite.com/elastic/kibana-single-user-performance/builds/12552 Before: image After Screenshot 2024-02-02 at 13 56 23 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/pipelines/performance/daily.yml | 6 - .../functional/performance_playwright.sh | 9 + .../src/functional_tests/run_tests/flags.ts | 12 +- src/dev/performance/run_performance_cli.ts | 215 ++++++++++-------- src/dev/tsconfig.json | 1 + 5 files changed, 138 insertions(+), 105 deletions(-) diff --git a/.buildkite/pipelines/performance/daily.yml b/.buildkite/pipelines/performance/daily.yml index 8a28ed2a561c3..a58deb281d2c5 100644 --- a/.buildkite/pipelines/performance/daily.yml +++ b/.buildkite/pipelines/performance/daily.yml @@ -22,12 +22,6 @@ steps: depends_on: build key: tests timeout_in_minutes: 90 - retry: - automatic: - - exit_status: '-1' - limit: 2 - - exit_status: '*' - limit: 1 - label: '📈 Report performance metrics to ci-stats' command: .buildkite/scripts/steps/functional/report_performance_metrics.sh diff --git a/.buildkite/scripts/steps/functional/performance_playwright.sh b/.buildkite/scripts/steps/functional/performance_playwright.sh index 5c8e86f48506b..de92e00092b17 100644 --- a/.buildkite/scripts/steps/functional/performance_playwright.sh +++ b/.buildkite/scripts/steps/functional/performance_playwright.sh @@ -30,3 +30,12 @@ if [ -d "$JOURNEY_SCREENSHOTS_DIR" ]; then buildkite-agent artifact upload "**/*fullscreen*.png" cd "$KIBANA_DIR" fi + +echo "--- Upload server logs" +JOURNEY_SERVER_LOGS_REL_PATH=".ftr/journey_server_logs" +JOURNEY_SERVER_LOGS_DIR="${KIBANA_DIR}/${JOURNEY_SERVER_LOGS_REL_PATH}" +if [ -d "$JOURNEY_SERVER_LOGS_DIR" ]; then + cd "$KIBANA_DIR" + tar -czf server-logs.tar.gz $JOURNEY_SERVER_LOGS_REL_PATH/**/* + buildkite-agent artifact upload server-logs.tar.gz +fi diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.ts index 3ba86999d3802..22253cdbbc8b8 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.ts @@ -30,6 +30,7 @@ export const FLAG_OPTIONS: FlagOptions = { 'exclude-tag', 'include', 'exclude', + 'writeLogsToPath', ], alias: { updateAll: 'u', @@ -47,6 +48,7 @@ export const FLAG_OPTIONS: FlagOptions = { --kibana-install-dir Run Kibana from existing install directory instead of from source --bail Stop the test run at the first failure --logToFile Write the log output from Kibana/ES to files instead of to stdout + --writeLogsToPath Write the log output from Kibana/ES to files in specified path --dry-run Report tests without executing them --updateBaselines Replace baseline screenshots with whatever is generated from the test --updateSnapshots Replace inline and file snapshots with whatever is generated from the test @@ -73,6 +75,12 @@ export function parseFlags(flags: FlagsReader) { const esVersionString = flags.string('es-version'); + const logsDir = flags.path('writeLogsToPath') + ? Path.resolve(REPO_ROOT, flags.path('writeLogsToPath')!) + : flags.boolean('logToFile') + ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) + : undefined; + return { configs, esVersion: esVersionString ? new EsVersion(esVersionString) : EsVersion.getDefault(), @@ -80,9 +88,7 @@ export function parseFlags(flags: FlagsReader) { dryRun: flags.boolean('dry-run'), updateBaselines: flags.boolean('updateBaselines') || flags.boolean('updateAll'), updateSnapshots: flags.boolean('updateSnapshots') || flags.boolean('updateAll'), - logsDir: flags.boolean('logToFile') - ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) - : undefined, + logsDir, esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']), esServerlessImage: flags.string('esServerlessImage'), installDir: flags.path('kibana-install-dir'), diff --git a/src/dev/performance/run_performance_cli.ts b/src/dev/performance/run_performance_cli.ts index 687a6460d5a3d..810827a89f73f 100644 --- a/src/dev/performance/run_performance_cli.ts +++ b/src/dev/performance/run_performance_cli.ts @@ -8,7 +8,9 @@ import { createFlagError } from '@kbn/dev-cli-errors'; import { run } from '@kbn/dev-cli-runner'; +import { ProcRunner } from '@kbn/dev-proc-runner'; import { REPO_ROOT } from '@kbn/repo-info'; +import { ToolingLog } from '@kbn/tooling-log'; import fs from 'fs'; import path from 'path'; @@ -19,6 +21,92 @@ export interface Journey { path: string; } +interface EsRunProps { + procRunner: ProcRunner; + log: ToolingLog; + logsDir?: string; +} + +interface TestRunProps extends EsRunProps { + journey: Journey; + phase: 'TEST' | 'WARMUP'; + kibanaInstallDir: string | undefined; +} + +const readFilesRecursively = (dir: string, callback: Function) => { + const files = fs.readdirSync(dir); + files.forEach((file) => { + const filePath = path.join(dir, file); + const stat = fs.statSync(filePath); + if (stat.isDirectory()) { + readFilesRecursively(filePath, callback); + } else if (stat.isFile()) { + callback(filePath); + } + }); +}; + +async function startEs(props: EsRunProps) { + const { procRunner, log, logsDir } = props; + await procRunner.run('es', { + cmd: 'node', + args: [ + 'scripts/es', + 'snapshot', + '--license=trial', + // Temporarily disabling APM + // ...(JOURNEY_APM_CONFIG.active + // ? [ + // '-E', + // 'tracing.apm.enabled=true', + // '-E', + // 'tracing.apm.agent.transaction_sample_rate=1.0', + // '-E', + // `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`, + // '-E', + // `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`, + // '-E', + // `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`, + // ] + // : []), + `--writeLogsToPath=${logsDir}/es-cluster.log`, + ], + cwd: REPO_ROOT, + wait: /kbn\/es setup complete/, + }); + + log.info(`✅ ES is ready and will run in the background`); +} + +async function runFunctionalTest(props: TestRunProps) { + const { procRunner, journey, phase, kibanaInstallDir, logsDir } = props; + await procRunner.run('functional-tests', { + cmd: 'node', + args: [ + 'scripts/functional_tests', + ['--config', journey.path], + kibanaInstallDir ? ['--kibana-install-dir', kibanaInstallDir] : [], + // save Kibana logs in file instead of console output; only for "warmup" phase + logsDir ? ['--writeLogsToPath', logsDir] : [], + '--debug', + '--bail', + ].flat(), + cwd: REPO_ROOT, + wait: true, + env: { + // Reset all the ELASTIC APM env vars to undefined, FTR config might set it's own values. + ...Object.fromEntries( + Object.keys(process.env).flatMap((k) => + k.startsWith('ELASTIC_APM_') ? [[k, undefined]] : [] + ) + ), + TEST_PERFORMANCE_PHASE: phase, + TEST_ES_URL: 'http://elastic:changeme@localhost:9200', + TEST_ES_DISABLE_STARTUP: 'true', + }, + }); +} + run( async ({ log, flagsReader, procRunner }) => { const skipWarmup = flagsReader.boolean('skip-warmup'); @@ -33,19 +121,23 @@ run( throw createFlagError('--journey-path must be an existing path'); } - let journeys: Journey[] = []; + const journeys: Journey[] = []; - if (journeyPath) { - journeys = fs.statSync(journeyPath).isDirectory() - ? fs.readdirSync(journeyPath).map((fileName) => { - return { name: fileName, path: path.resolve(journeyPath, fileName) }; - }) - : [{ name: path.parse(journeyPath).name, path: journeyPath }]; + if (journeyPath && fs.statSync(journeyPath).isFile()) { + journeys.push({ name: path.parse(journeyPath).name, path: journeyPath }); } else { - const journeyBasePath = path.resolve(REPO_ROOT, JOURNEY_BASE_PATH); - journeys = fs.readdirSync(journeyBasePath).map((name) => { - return { name, path: path.join(journeyBasePath, name) }; - }); + // default dir is x-pack/performance/journeys + const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH); + readFilesRecursively(dir, (filePath: string) => + journeys.push({ + name: path.parse(filePath).name, + path: path.resolve(dir, filePath), + }) + ); + } + + if (journeys.length === 0) { + throw new Error('No journeys found'); } log.info( @@ -56,11 +148,24 @@ run( for (const journey of journeys) { try { - await startEs(); + // create folder to store ES/Kibana server logs + const logsDir = path.resolve(REPO_ROOT, `.ftr/journey_server_logs/${journey.name}`); + fs.mkdirSync(logsDir, { recursive: true }); + process.stdout.write(`--- Running journey: ${journey.name} [start ES, warmup run]\n`); + await startEs({ procRunner, log, logsDir }); if (!skipWarmup) { - await runWarmup(journey, kibanaInstallDir); + // Set the phase to WARMUP, this will prevent the FTR from starting Kibana with opt-in telemetry and save logs to file + await runFunctionalTest({ + procRunner, + log, + journey, + phase: 'WARMUP', + kibanaInstallDir, + logsDir, + }); } - await runTest(journey, kibanaInstallDir); + process.stdout.write(`--- Running journey: ${journey.name} [collect metrics]\n`); + await runFunctionalTest({ procRunner, log, journey, phase: 'TEST', kibanaInstallDir }); } catch (e) { log.error(e); failedJourneys.push(journey.name); @@ -69,88 +174,6 @@ run( } } - async function runFunctionalTest( - configPath: string, - phase: 'TEST' | 'WARMUP', - kibanaBuildDir: string | undefined - ) { - await procRunner.run('functional-tests', { - cmd: 'node', - args: [ - 'scripts/functional_tests', - ['--config', configPath], - kibanaBuildDir ? ['--kibana-install-dir', kibanaBuildDir] : [], - '--debug', - '--bail', - ].flat(), - cwd: REPO_ROOT, - wait: true, - env: { - // Reset all the ELASTIC APM env vars to undefined, FTR config might set it's own values. - ...Object.fromEntries( - Object.keys(process.env).flatMap((k) => - k.startsWith('ELASTIC_APM_') ? [[k, undefined]] : [] - ) - ), - TEST_PERFORMANCE_PHASE: phase, - TEST_ES_URL: 'http://elastic:changeme@localhost:9200', - TEST_ES_DISABLE_STARTUP: 'true', - }, - }); - } - - async function startEs() { - process.stdout.write(`--- Starting ES\n`); - await procRunner.run('es', { - cmd: 'node', - args: [ - 'scripts/es', - 'snapshot', - '--license=trial', - // Temporarily disabling APM - // ...(JOURNEY_APM_CONFIG.active - // ? [ - // '-E', - // 'tracing.apm.enabled=true', - // '-E', - // 'tracing.apm.agent.transaction_sample_rate=1.0', - // '-E', - // `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`, - // '-E', - // `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`, - // '-E', - // `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`, - // ] - // : []), - ], - cwd: REPO_ROOT, - wait: /kbn\/es setup complete/, - }); - - log.info(`✅ ES is ready and will run in the background`); - } - - async function runWarmup(journey: Journey, kibanaBuildDir: string | undefined) { - try { - process.stdout.write(`--- Running warmup: ${journey.name}\n`); - // Set the phase to WARMUP, this will prevent the functional test server from starting Elasticsearch, opt in to telemetry, etc. - await runFunctionalTest(journey.path, 'WARMUP', kibanaBuildDir); - } catch (e) { - log.warning(`Warmup for ${journey.name} failed`); - throw e; - } - } - - async function runTest(journey: Journey, kibanaBuildDir: string | undefined) { - try { - process.stdout.write(`--- Running ${journey.name}\n`); - await runFunctionalTest(journey.path, 'TEST', kibanaBuildDir); - } catch (e) { - log.warning(`Journey ${journey.name} failed. Retrying once...`); - await runFunctionalTest(journey.path, 'TEST', kibanaBuildDir); - } - } - if (failedJourneys.length > 0) { throw new Error(`${failedJourneys.length} journeys failed: ${failedJourneys.join(',')}`); } diff --git a/src/dev/tsconfig.json b/src/dev/tsconfig.json index a3ebcf6dda1ba..ca2253b2ee055 100644 --- a/src/dev/tsconfig.json +++ b/src/dev/tsconfig.json @@ -41,5 +41,6 @@ "@kbn/config-schema", "@kbn/core-test-helpers-so-type-serializer", "@kbn/core-test-helpers-kbn-server", + "@kbn/dev-proc-runner", ] }