forked from elastic/kibana
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] Comment flaky test results on tested PR (elastic#183043)
## Summary Extends the flaky-test-runner with the capability to comment on the flaky test runs on the PR that's being tested. Closes: elastic#173129 - chore(flaky-test-runner): Add a step to collect results and comment on the tested PR (cherry picked from commit 38d4230)
- Loading branch information
Showing
3 changed files
with
152 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| import { BuildkiteClient, getGithubClient } from '#pipeline-utils'; | ||
|
|
||
| interface TestSuiteResult { | ||
| name: string; | ||
| success: boolean; | ||
| successCount: number; | ||
| groupSize: number; | ||
| } | ||
|
|
||
| async function main() { | ||
| // Get buildkite build | ||
| const buildkite = new BuildkiteClient(); | ||
| const buildkiteBuild = await buildkite.getBuild( | ||
| process.env.BUILDKITE_PIPELINE_SLUG!, | ||
| process.env.BUILDKITE_BUILD_NUMBER! | ||
| ); | ||
| const buildLink = `[${buildkiteBuild.pipeline.slug}#${buildkiteBuild.number}](${buildkiteBuild.web_url})`; | ||
|
|
||
| // Calculate success metrics | ||
| const jobs = buildkiteBuild.jobs; | ||
| const testSuiteRuns = jobs.filter((step) => { | ||
| return step.step_key?.includes('ftr-suite') || step.step_key?.includes('cypress-suite'); | ||
| }); | ||
| const testSuiteGroups = groupBy('name', testSuiteRuns); | ||
|
|
||
| const success = testSuiteRuns.every((job) => job.state === 'passed'); | ||
| const testGroupResults = Object.entries(testSuiteGroups).map(([name, group]) => { | ||
| const passingTests = group.filter((job) => job.state === 'passed'); | ||
| return { | ||
| name, | ||
| success: passingTests.length === group.length, | ||
| successCount: passingTests.length, | ||
| groupSize: group.length, | ||
| }; | ||
| }); | ||
|
|
||
| // Comment results on the PR | ||
| const prNumber = Number(extractPRNumberFromBranch(buildkiteBuild.branch)); | ||
| if (isNaN(prNumber)) { | ||
| throw new Error(`Couldn't find PR number for build ${buildkiteBuild.web_url}.`); | ||
| } | ||
| const flakyRunHistoryLink = `https://buildkite.com/elastic/${ | ||
| buildkiteBuild.pipeline.slug | ||
| }/builds?branch=${encodeURIComponent(buildkiteBuild.branch)}`; | ||
|
|
||
| const prComment = ` | ||
| ## Flaky Test Runner Stats | ||
| ### ${success ? '🎉 All tests passed!' : '🟠 Some tests failed.'} - ${buildLink} | ||
| ${testGroupResults.map(formatTestGroupResult).join('\n')} | ||
| [see run history](${flakyRunHistoryLink}) | ||
| `; | ||
|
|
||
| const githubClient = getGithubClient(); | ||
| const commentResult = await githubClient.issues.createComment({ | ||
| owner: 'elastic', | ||
| repo: 'kibana', | ||
| body: prComment, | ||
| issue_number: prNumber, | ||
| }); | ||
|
|
||
| console.log(`Comment added: ${commentResult.data.html_url}`); | ||
| } | ||
|
|
||
| function formatTestGroupResult(result: TestSuiteResult) { | ||
| const statusIcon = result.success ? '✅' : '❌'; | ||
| const testName = result.name; | ||
| const successCount = result.successCount; | ||
| const groupSize = result.groupSize; | ||
|
|
||
| return `[${statusIcon}] ${testName}: ${successCount}/${groupSize} tests passed.`; | ||
| } | ||
|
|
||
| function groupBy<T>(field: keyof T, values: T[]): Record<string, T[]> { | ||
| return values.reduce((acc, value) => { | ||
| const key = value[field]; | ||
| if (typeof key !== 'string') { | ||
| throw new Error('Cannot group by non-string value field'); | ||
| } | ||
|
|
||
| if (!acc[key]) { | ||
| acc[key] = []; | ||
| } | ||
| acc[key].push(value); | ||
| return acc; | ||
| }, {} as Record<string, T[]>); | ||
| } | ||
|
|
||
| function extractPRNumberFromBranch(branch: string | undefined) { | ||
| if (!branch) { | ||
| return null; | ||
| } else { | ||
| return branch.match(/refs\/pull\/(\d+)\/head/)?.[1]; | ||
| } | ||
| } | ||
|
|
||
| main() | ||
| .then(() => { | ||
| console.log('Flaky runner stats comment added to PR!'); | ||
| }) | ||
| .catch((e) => { | ||
| console.error(e); | ||
| process.exit(1); | ||
| }); |