Skip to content

Commit

Permalink
[kbn-journey] improve CI output by saving server logs into files (ela…
Browse files Browse the repository at this point in the history 
…stic#175293)

## Summary

This PR refactors `run_performance` script logic that we use to run
single user performance journeys:
- simplify script by keeping only 2 functions: `startES` &
`runFunctionalTest`
- write ES and Kibana logs (for warmup phase) to files instead of
console output;
- upload files with server log as build artifacts
- do not retry journey TEST phase run on failure, it doubles EBT metrics
and does not provide much value: journey should be simple and stable.
- do not retry step running journeys, most of the failures are due to
journey flakiness and we just skip them

Testing performance pipeline
https://buildkite.com/elastic/kibana-single-user-performance/builds/12552


Before: 
<img width="1243" alt="image"
src="https://github.com/elastic/kibana/assets/10977896/c4635986-6552-4842-abf9-640f09630674">

After
<img width="1243" alt="Screenshot 2024-02-02 at 13 56 23"
src="https://github.com/elastic/kibana/assets/10977896/47ff6a76-d8dd-44dd-93a5-2c49ef87e808">

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
@dmlemeshko @kibanamachine @fkanout
2 people authored and fkanout committed Mar 4, 2024
1 parent 827cdfc commit 3cf8077
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 105 deletions.
6 changes: 0 additions & 6 deletions .buildkite/pipelines/performance/daily.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,6 @@ steps:
depends_on: build
key: tests
timeout_in_minutes: 90
retry:
automatic:
- exit_status: '-1'
limit: 2
- exit_status: '*'
limit: 1

- label: '📈 Report performance metrics to ci-stats'
command: .buildkite/scripts/steps/functional/report_performance_metrics.sh
Expand Down
9 changes: 9 additions & 0 deletions .buildkite/scripts/steps/functional/performance_playwright.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,12 @@ if [ -d "$JOURNEY_SCREENSHOTS_DIR" ]; then
buildkite-agent artifact upload "**/*fullscreen*.png"
cd "$KIBANA_DIR"
fi

echo "--- Upload server logs"
JOURNEY_SERVER_LOGS_REL_PATH=".ftr/journey_server_logs"
JOURNEY_SERVER_LOGS_DIR="${KIBANA_DIR}/${JOURNEY_SERVER_LOGS_REL_PATH}"
if [ -d "$JOURNEY_SERVER_LOGS_DIR" ]; then
cd "$KIBANA_DIR"
tar -czf server-logs.tar.gz $JOURNEY_SERVER_LOGS_REL_PATH/**/*
buildkite-agent artifact upload server-logs.tar.gz
fi
12 changes: 9 additions & 3 deletions packages/kbn-test/src/functional_tests/run_tests/flags.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export const FLAG_OPTIONS: FlagOptions = {
'exclude-tag',
'include',
'exclude',
'writeLogsToPath',
],
alias: {
updateAll: 'u',
Expand All @@ -47,6 +48,7 @@ export const FLAG_OPTIONS: FlagOptions = {
--kibana-install-dir Run Kibana from existing install directory instead of from source
--bail Stop the test run at the first failure
--logToFile Write the log output from Kibana/ES to files instead of to stdout
--writeLogsToPath Write the log output from Kibana/ES to files in specified path
--dry-run Report tests without executing them
--updateBaselines Replace baseline screenshots with whatever is generated from the test
--updateSnapshots Replace inline and file snapshots with whatever is generated from the test
Expand All @@ -73,16 +75,20 @@ export function parseFlags(flags: FlagsReader) {

const esVersionString = flags.string('es-version');

const logsDir = flags.path('writeLogsToPath')
? Path.resolve(REPO_ROOT, flags.path('writeLogsToPath')!)
: flags.boolean('logToFile')
? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4())
: undefined;

return {
configs,
esVersion: esVersionString ? new EsVersion(esVersionString) : EsVersion.getDefault(),
bail: flags.boolean('bail'),
dryRun: flags.boolean('dry-run'),
updateBaselines: flags.boolean('updateBaselines') || flags.boolean('updateAll'),
updateSnapshots: flags.boolean('updateSnapshots') || flags.boolean('updateAll'),
logsDir: flags.boolean('logToFile')
? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4())
: undefined,
logsDir,
esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']),
esServerlessImage: flags.string('esServerlessImage'),
installDir: flags.path('kibana-install-dir'),
Expand Down
215 changes: 119 additions & 96 deletions src/dev/performance/run_performance_cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@

import { createFlagError } from '@kbn/dev-cli-errors';
import { run } from '@kbn/dev-cli-runner';
import { ProcRunner } from '@kbn/dev-proc-runner';
import { REPO_ROOT } from '@kbn/repo-info';
import { ToolingLog } from '@kbn/tooling-log';
import fs from 'fs';
import path from 'path';

Expand All @@ -19,6 +21,92 @@ export interface Journey {
path: string;
}

interface EsRunProps {
procRunner: ProcRunner;
log: ToolingLog;
logsDir?: string;
}

interface TestRunProps extends EsRunProps {
journey: Journey;
phase: 'TEST' | 'WARMUP';
kibanaInstallDir: string | undefined;
}

const readFilesRecursively = (dir: string, callback: Function) => {
const files = fs.readdirSync(dir);
files.forEach((file) => {
const filePath = path.join(dir, file);
const stat = fs.statSync(filePath);
if (stat.isDirectory()) {
readFilesRecursively(filePath, callback);
} else if (stat.isFile()) {
callback(filePath);
}
});
};

async function startEs(props: EsRunProps) {
const { procRunner, log, logsDir } = props;
await procRunner.run('es', {
cmd: 'node',
args: [
'scripts/es',
'snapshot',
'--license=trial',
// Temporarily disabling APM
// ...(JOURNEY_APM_CONFIG.active
// ? [
// '-E',
// 'tracing.apm.enabled=true',
// '-E',
// 'tracing.apm.agent.transaction_sample_rate=1.0',
// '-E',
// `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
// '-E',
// `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
// '-E',
// `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
// ]
// : []),
`--writeLogsToPath=${logsDir}/es-cluster.log`,
],
cwd: REPO_ROOT,
wait: /kbn\/es setup complete/,
});

log.info(`✅ ES is ready and will run in the background`);
}

async function runFunctionalTest(props: TestRunProps) {
const { procRunner, journey, phase, kibanaInstallDir, logsDir } = props;
await procRunner.run('functional-tests', {
cmd: 'node',
args: [
'scripts/functional_tests',
['--config', journey.path],
kibanaInstallDir ? ['--kibana-install-dir', kibanaInstallDir] : [],
// save Kibana logs in file instead of console output; only for "warmup" phase
logsDir ? ['--writeLogsToPath', logsDir] : [],
'--debug',
'--bail',
].flat(),
cwd: REPO_ROOT,
wait: true,
env: {
// Reset all the ELASTIC APM env vars to undefined, FTR config might set it's own values.
...Object.fromEntries(
Object.keys(process.env).flatMap((k) =>
k.startsWith('ELASTIC_APM_') ? [[k, undefined]] : []
)
),
TEST_PERFORMANCE_PHASE: phase,
TEST_ES_URL: 'http://elastic:changeme@localhost:9200',
TEST_ES_DISABLE_STARTUP: 'true',
},
});
}

run(
async ({ log, flagsReader, procRunner }) => {
const skipWarmup = flagsReader.boolean('skip-warmup');
Expand All @@ -33,19 +121,23 @@ run(
throw createFlagError('--journey-path must be an existing path');
}

let journeys: Journey[] = [];
const journeys: Journey[] = [];

if (journeyPath) {
journeys = fs.statSync(journeyPath).isDirectory()
? fs.readdirSync(journeyPath).map((fileName) => {
return { name: fileName, path: path.resolve(journeyPath, fileName) };
})
: [{ name: path.parse(journeyPath).name, path: journeyPath }];
if (journeyPath && fs.statSync(journeyPath).isFile()) {
journeys.push({ name: path.parse(journeyPath).name, path: journeyPath });
} else {
const journeyBasePath = path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
journeys = fs.readdirSync(journeyBasePath).map((name) => {
return { name, path: path.join(journeyBasePath, name) };
});
// default dir is x-pack/performance/journeys
const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH);
readFilesRecursively(dir, (filePath: string) =>
journeys.push({
name: path.parse(filePath).name,
path: path.resolve(dir, filePath),
})
);
}

if (journeys.length === 0) {
throw new Error('No journeys found');
}

log.info(
Expand All @@ -56,11 +148,24 @@ run(

for (const journey of journeys) {
try {
await startEs();
// create folder to store ES/Kibana server logs
const logsDir = path.resolve(REPO_ROOT, `.ftr/journey_server_logs/${journey.name}`);
fs.mkdirSync(logsDir, { recursive: true });
process.stdout.write(`--- Running journey: ${journey.name} [start ES, warmup run]\n`);
await startEs({ procRunner, log, logsDir });
if (!skipWarmup) {
await runWarmup(journey, kibanaInstallDir);
// Set the phase to WARMUP, this will prevent the FTR from starting Kibana with opt-in telemetry and save logs to file
await runFunctionalTest({
procRunner,
log,
journey,
phase: 'WARMUP',
kibanaInstallDir,
logsDir,
});
}
await runTest(journey, kibanaInstallDir);
process.stdout.write(`--- Running journey: ${journey.name} [collect metrics]\n`);
await runFunctionalTest({ procRunner, log, journey, phase: 'TEST', kibanaInstallDir });
} catch (e) {
log.error(e);
failedJourneys.push(journey.name);
Expand All @@ -69,88 +174,6 @@ run(
}
}

async function runFunctionalTest(
configPath: string,
phase: 'TEST' | 'WARMUP',
kibanaBuildDir: string | undefined
) {
await procRunner.run('functional-tests', {
cmd: 'node',
args: [
'scripts/functional_tests',
['--config', configPath],
kibanaBuildDir ? ['--kibana-install-dir', kibanaBuildDir] : [],
'--debug',
'--bail',
].flat(),
cwd: REPO_ROOT,
wait: true,
env: {
// Reset all the ELASTIC APM env vars to undefined, FTR config might set it's own values.
...Object.fromEntries(
Object.keys(process.env).flatMap((k) =>
k.startsWith('ELASTIC_APM_') ? [[k, undefined]] : []
)
),
TEST_PERFORMANCE_PHASE: phase,
TEST_ES_URL: 'http://elastic:changeme@localhost:9200',
TEST_ES_DISABLE_STARTUP: 'true',
},
});
}

async function startEs() {
process.stdout.write(`--- Starting ES\n`);
await procRunner.run('es', {
cmd: 'node',
args: [
'scripts/es',
'snapshot',
'--license=trial',
// Temporarily disabling APM
// ...(JOURNEY_APM_CONFIG.active
// ? [
// '-E',
// 'tracing.apm.enabled=true',
// '-E',
// 'tracing.apm.agent.transaction_sample_rate=1.0',
// '-E',
// `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
// '-E',
// `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
// '-E',
// `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
// ]
// : []),
],
cwd: REPO_ROOT,
wait: /kbn\/es setup complete/,
});

log.info(`✅ ES is ready and will run in the background`);
}

async function runWarmup(journey: Journey, kibanaBuildDir: string | undefined) {
try {
process.stdout.write(`--- Running warmup: ${journey.name}\n`);
// Set the phase to WARMUP, this will prevent the functional test server from starting Elasticsearch, opt in to telemetry, etc.
await runFunctionalTest(journey.path, 'WARMUP', kibanaBuildDir);
} catch (e) {
log.warning(`Warmup for ${journey.name} failed`);
throw e;
}
}

async function runTest(journey: Journey, kibanaBuildDir: string | undefined) {
try {
process.stdout.write(`--- Running ${journey.name}\n`);
await runFunctionalTest(journey.path, 'TEST', kibanaBuildDir);
} catch (e) {
log.warning(`Journey ${journey.name} failed. Retrying once...`);
await runFunctionalTest(journey.path, 'TEST', kibanaBuildDir);
}
}

if (failedJourneys.length > 0) {
throw new Error(`${failedJourneys.length} journeys failed: ${failedJourneys.join(',')}`);
}
Expand Down
1 change: 1 addition & 0 deletions src/dev/tsconfig.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,6 @@
"@kbn/config-schema",
"@kbn/core-test-helpers-so-type-serializer",
"@kbn/core-test-helpers-kbn-server",
"@kbn/dev-proc-runner",
]
}

0 comments on commit 3cf8077

Please sign in to comment.