From af485dada1894756f2b5271e2a0ae01d51acfc78 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 9 Dec 2024 10:39:39 -0700 Subject: [PATCH 1/3] feat: junit result formatter --- messages/shared.md | 8 ++++ src/commands/agent/test/results.ts | 16 +++++--- src/commands/agent/test/resume.ts | 16 +++++--- src/commands/agent/test/run.ts | 17 ++++++--- src/flags.ts | 8 +++- src/handleTestResults.ts | 61 ++++++++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 src/handleTestResults.ts diff --git a/messages/shared.md b/messages/shared.md index af9268c..a9c7f45 100644 --- a/messages/shared.md +++ b/messages/shared.md @@ -1,3 +1,11 @@ # flags.result-format.summary Format of the test run results. + +# flags.output-dir.summary + +Directory to write the test results to. + +# flags.output-dir.description + +If test run is complete, write the results to the specified directory. If the tests are still running, the test results will not be written. diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 5443fc4..7fb2ef5 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -7,8 +7,9 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester, AgentTestDetailsResponse, humanFormat } from '@salesforce/agents'; -import { resultFormatFlag } from '../../../flags.js'; +import { AgentTester, AgentTestDetailsResponse } from '@salesforce/agents'; +import { resultFormatFlag, testOutputDirFlag } from '../../../flags.js'; +import { handleTestResults } from '../../../handleTestResults.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.results'); @@ -30,6 +31,7 @@ export default class AgentTestResults extends SfCommand required: true, }), 'result-format': resultFormatFlag(), + 'output-dir': testOutputDirFlag(), }; public async run(): Promise { @@ -37,9 +39,13 @@ export default class AgentTestResults extends SfCommand const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); const response = await agentTester.details(flags['job-id']); - if (flags['result-format'] === 'human') { - this.log(await humanFormat(flags['job-id'], response)); - } + await handleTestResults({ + id: flags['job-id'], + format: flags['result-format'], + results: response, + jsonEnabled: this.jsonEnabled(), + outputDir: flags['output-dir'], + }); return response; } } diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 2087612..4fbd2de 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -7,10 +7,11 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester, humanFormat } from '@salesforce/agents'; +import { AgentTester } from '@salesforce/agents'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; -import { resultFormatFlag } from '../../../flags.js'; +import { resultFormatFlag, testOutputDirFlag } from '../../../flags.js'; +import { handleTestResults } from '../../../handleTestResults.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); @@ -48,6 +49,7 @@ export default class AgentTestResume extends SfCommand { description: messages.getMessage('flags.wait.description'), }), 'result-format': resultFormatFlag(), + 'output-dir': testOutputDirFlag(), }; public async run(): Promise { @@ -68,9 +70,13 @@ export default class AgentTestResume extends SfCommand { mso.stop(); - if (response && flags['result-format'] === 'human') { - this.log(await humanFormat(name ?? aiEvaluationId, response)); - } + await handleTestResults({ + id: aiEvaluationId, + format: flags['result-format'], + results: response, + jsonEnabled: this.jsonEnabled(), + outputDir: flags['output-dir'], + }); return { status: 'COMPLETED', diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 1b2a2c9..51f9da5 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -7,11 +7,12 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester, humanFormat } from '@salesforce/agents'; +import { AgentTester } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; -import { resultFormatFlag } from '../../../flags.js'; +import { resultFormatFlag, testOutputDirFlag } from '../../../flags.js'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; +import { handleTestResults } from '../../../handleTestResults.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); @@ -47,6 +48,7 @@ export default class AgentTestRun extends SfCommand { description: messages.getMessage('flags.wait.description'), }), 'result-format': resultFormatFlag(), + 'output-dir': testOutputDirFlag(), }; public async run(): Promise { @@ -69,9 +71,14 @@ export default class AgentTestRun extends SfCommand { mso.stop(); - if (detailsResponse && flags['result-format'] === 'human') { - this.log(await humanFormat(flags.name, detailsResponse)); - } + await handleTestResults({ + id: response.aiEvaluationId, + format: flags['result-format'], + results: detailsResponse, + jsonEnabled: this.jsonEnabled(), + outputDir: flags['output-dir'], + }); + return { status: 'COMPLETED', aiEvaluationId: response.aiEvaluationId, diff --git a/src/flags.ts b/src/flags.ts index 418e308..0478cbf 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -14,9 +14,15 @@ export const resultFormatFlag = Flags.option({ options: [ 'json', 'human', + 'junit', // 'tap', - // 'junit' ] as const, default: 'human', summary: messages.getMessage('flags.result-format.summary'), }); + +export const testOutputDirFlag = Flags.custom({ + char: 'f', + description: messages.getMessage('flags.output-dir.description'), + summary: messages.getMessage('flags.output-dir.summary'), +}); diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts new file mode 100644 index 0000000..38b784d --- /dev/null +++ b/src/handleTestResults.ts @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { writeFile, mkdir } from 'node:fs/promises'; +import { AgentTestDetailsResponse, jsonFormat, humanFormat, junitFormat } from '@salesforce/agents'; +import { Ux } from '@salesforce/sf-plugins-core/Ux'; + +async function writeFileToDir(outputDir: string, fileName: string, content: string): Promise { + // if directory doesn't exist, create it + await mkdir(outputDir, { recursive: true }); + + await writeFile(`${outputDir}/${fileName}`, content); +} + +export async function handleTestResults({ + id, + format, + results, + jsonEnabled, + outputDir, +}: { + id: string; + format: 'human' | 'json' | 'junit'; + results: AgentTestDetailsResponse | undefined; + jsonEnabled: boolean; + outputDir?: string; +}): Promise { + if (!results) { + // do nothing since there are no results to handle + return; + } + + const ux = new Ux({ jsonEnabled }); + + if (format === 'human') { + const formatted = await humanFormat(results); + ux.log(formatted); + if (outputDir) { + await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted); + } + } + + if (format === 'json') { + const formatted = await jsonFormat(results); + ux.log(formatted); + if (outputDir) { + await writeFileToDir(outputDir, `test-result-${id}.json`, formatted); + } + } + + if (format === 'junit') { + const formatted = await junitFormat(results); + ux.log(formatted); + if (outputDir) { + await writeFileToDir(outputDir, `test-result-${id}.xml`, formatted); + } + } +} From 2c3596fec39ef88b591fb57868c899d93e653289 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 9 Dec 2024 10:41:14 -0700 Subject: [PATCH 2/3] chore: update snapshot --- command-snapshot.json | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index 12adfde..bf66e8c 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -47,24 +47,34 @@ "alias": [], "command": "agent:test:results", "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "job-id", "json", "result-format", "target-org"], + "flagChars": ["f", "i", "o"], + "flags": ["api-version", "flags-dir", "job-id", "json", "output-dir", "result-format", "target-org"], "plugin": "@salesforce/plugin-agent" }, { "alias": [], "command": "agent:test:resume", "flagAliases": [], - "flagChars": ["i", "o", "r", "w"], - "flags": ["api-version", "flags-dir", "job-id", "json", "result-format", "target-org", "use-most-recent", "wait"], + "flagChars": ["f", "i", "o", "r", "w"], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "use-most-recent", + "wait" + ], "plugin": "@salesforce/plugin-agent" }, { "alias": [], "command": "agent:test:run", "flagAliases": [], - "flagChars": ["n", "o", "w"], - "flags": ["api-version", "flags-dir", "json", "name", "result-format", "target-org", "wait"], + "flagChars": ["f", "n", "o", "w"], + "flags": ["api-version", "flags-dir", "json", "name", "output-dir", "result-format", "target-org", "wait"], "plugin": "@salesforce/plugin-agent" } ] From 191495688cadd1b6e809c28fdfef03d6763c7527 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Tue, 10 Dec 2024 12:09:48 -0700 Subject: [PATCH 3/3] chore: code review --- package.json | 2 +- src/handleTestResults.ts | 3 +- yarn.lock | 83 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 9fa91f6..5cd8190 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "@inquirer/select": "^4.0.1", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.7.12", - "@salesforce/agents": "^0.3.0", + "@salesforce/agents": "^0.4.0", "@salesforce/core": "^8.8.0", "@salesforce/kit": "^3.2.1", "@salesforce/sf-plugins-core": "^12.1.0", diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 38b784d..0352916 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -4,6 +4,7 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { join } from 'node:path'; import { writeFile, mkdir } from 'node:fs/promises'; import { AgentTestDetailsResponse, jsonFormat, humanFormat, junitFormat } from '@salesforce/agents'; import { Ux } from '@salesforce/sf-plugins-core/Ux'; @@ -12,7 +13,7 @@ async function writeFileToDir(outputDir: string, fileName: string, content: stri // if directory doesn't exist, create it await mkdir(outputDir, { recursive: true }); - await writeFile(`${outputDir}/${fileName}`, content); + await writeFile(join(outputDir, fileName), content); } export async function handleTestResults({ diff --git a/yarn.lock b/yarn.lock index cfdf5d6..f70c586 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1344,7 +1344,7 @@ http-call "^5.2.2" lodash "^4.17.21" -"@oclif/table@^0.3.2", "@oclif/table@^0.3.3": +"@oclif/table@^0.3.2": version "0.3.5" resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.3.5.tgz#118149eab364f3485eab5c9fd0d717c56082bacb" integrity sha512-1IjoVz7WAdUdBW5vYIRc6wt9N7Ezwll6AtdmeqLQ8lUmB9gQJVyeb7dqXtUaUvIG7bZMvryfPe6Xibeo5FTCWA== @@ -1360,6 +1360,22 @@ strip-ansi "^7.1.0" wrap-ansi "^9.0.0" +"@oclif/table@^0.3.5": + version "0.3.7" + resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.3.7.tgz#b13fb4525413c570cf1c34c8f6da9e13ea7ced0e" + integrity sha512-ixk/2swooqgg07N3mICe3gvkq/G4Au3jNMaRNCWIfWtcoyT+tTrOQxD+Ead0wMBuZl/J+CyEwImIjJqC21P1kA== + dependencies: + "@oclif/core" "^4" + "@types/react" "^18.3.12" + change-case "^5.4.4" + cli-truncate "^4.0.0" + ink "^5.1.0" + natural-orderby "^3.0.2" + object-hash "^3.0.0" + react "^18.3.1" + strip-ansi "^7.1.0" + wrap-ansi "^9.0.0" + "@oclif/test@^4.1.0": version "4.1.0" resolved "https://registry.yarnpkg.com/@oclif/test/-/test-4.1.0.tgz#7935e3707cf07480790139e02973196d18d16822" @@ -1373,15 +1389,16 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@salesforce/agents@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.3.0.tgz#5f58d69eca1dde07daaf88bc2226b1a09e579666" - integrity sha512-BV/Fa+WN8IT5n+bsdDI8wga5dxjY9Rhu6eAvU3OCyRQ7F0nFd5uqLe2Ybo+0gLbGCvGCrV9gt8eJ5z4fsgLoDQ== +"@salesforce/agents@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.4.0.tgz#7d2a1912113abf47bdb610db9352067a69747d24" + integrity sha512-H1CyEXyMkm9g55Zl55r5cL7ka4IQtoLqiDjfkxdoqEFn+FiQ6D2OpOR9ox2S5ATGLe1ELoAHWspncAcFBbMF/Q== dependencies: - "@oclif/table" "^0.3.3" + "@oclif/table" "^0.3.5" "@salesforce/core" "^8.8.0" "@salesforce/kit" "^3.2.3" "@salesforce/sf-plugins-core" "^12.1.0" + fast-xml-parser "^4" nock "^13.5.6" "@salesforce/cli-plugins-testkit@^5.3.35": @@ -3760,6 +3777,11 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" +es-toolkit@^1.22.0: + version "1.29.0" + resolved "https://registry.yarnpkg.com/es-toolkit/-/es-toolkit-1.29.0.tgz#a9c54e87880002e4186560fe0a567e76a61cb61a" + integrity sha512-GjTll+E6APcfAQA09D89HdT8Qn2Yb+TeDSDBTMcxAo+V+w1amAtCI15LJu4YPH/UCPoSo/F47Gr1LIM0TE0lZA== + es6-error@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" @@ -4151,6 +4173,13 @@ fast-xml-parser@4.4.1: dependencies: strnum "^1.0.5" +fast-xml-parser@^4: + version "4.5.0" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37" + integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg== + dependencies: + strnum "^1.0.5" + fastest-levenshtein@^1.0.7: version "1.0.16" resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5" @@ -4925,6 +4954,36 @@ ink@^5.0.1: ws "^8.15.0" yoga-wasm-web "~0.3.3" +ink@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/ink/-/ink-5.1.0.tgz#8ed050bf7a468489f231c99031f8bb1393c44079" + integrity sha512-3vIO+CU4uSg167/dZrg4wHy75llUINYXxN4OsdaCkE40q4zyOTPwNc2VEpLnnWsIvIQeo6x6lilAhuaSt+rIsA== + dependencies: + "@alcalzone/ansi-tokenize" "^0.1.3" + ansi-escapes "^7.0.0" + ansi-styles "^6.2.1" + auto-bind "^5.0.1" + chalk "^5.3.0" + cli-boxes "^3.0.0" + cli-cursor "^4.0.0" + cli-truncate "^4.0.0" + code-excerpt "^4.0.0" + es-toolkit "^1.22.0" + indent-string "^5.0.0" + is-in-ci "^1.0.0" + patch-console "^2.0.0" + react-reconciler "^0.29.0" + scheduler "^0.23.0" + signal-exit "^3.0.7" + slice-ansi "^7.1.0" + stack-utils "^2.0.6" + string-width "^7.2.0" + type-fest "^4.27.0" + widest-line "^5.0.0" + wrap-ansi "^9.0.0" + ws "^8.18.0" + yoga-wasm-web "~0.3.3" + internal-slot@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.5.tgz#f2a2ee21f668f8627a4667f309dc0f4fb6674986" @@ -5085,6 +5144,11 @@ is-in-ci@^0.1.0: resolved "https://registry.yarnpkg.com/is-in-ci/-/is-in-ci-0.1.0.tgz#5e07d6a02ec3a8292d3f590973357efa3fceb0d3" integrity sha512-d9PXLEY0v1iJ64xLiQMJ51J128EYHAaOR4yZqQi8aHGfw6KgifM3/Viw1oZZ1GCVmb3gBuyhLyHj0HgR2DhSXQ== +is-in-ci@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-in-ci/-/is-in-ci-1.0.0.tgz#9a86bbda7e42c6129902e0574c54b018fbb6ab88" + integrity sha512-eUuAjybVTHMYWm/U+vBO1sY/JOCgoPCXRxzdju0K+K0BiGW0SChEL1MLC0PoCIR1OlPo5YAp8HuQoUlsWEICwg== + is-map@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.3.tgz#ede96b7fe1e270b3c4465e3a465658764926d62e" @@ -7767,6 +7831,11 @@ type-fest@^4.18.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-4.29.1.tgz#9c125cb7c0cef6695f3c0b9d15d520c5dbadfcba" integrity sha512-Y1zUveI92UYM/vo1EFlQSsNf74+hfKH+7saZJslF0Fw92FRaiTAnHPIvo9d7SLxXt/gAYqA4RXyDTioMQCCp0A== +type-fest@^4.27.0: + version "4.30.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-4.30.0.tgz#cf411e7630578ad9e9884951dfaeef6588f970fe" + integrity sha512-G6zXWS1dLj6eagy6sVhOMQiLtJdxQBHIA9Z6HFUNLOlr6MFOgzV8wvmidtPONfPtEUv0uZsy77XJNzTAfwPDaA== + type-fest@^4.8.3: version "4.26.1" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-4.26.1.tgz#a4a17fa314f976dd3e6d6675ef6c775c16d7955e" @@ -8201,7 +8270,7 @@ write-file-atomic@^3.0.0: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" -ws@^8.15.0: +ws@^8.15.0, ws@^8.18.0: version "8.18.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==