From 42ff64bd855d4de5e4f6585ab5c141816d1bccf3 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 9 Dec 2024 10:38:52 -0700 Subject: [PATCH] feat: junit result formatter --- package.json | 1 + src/agentTester.ts | 61 ++++++++++++++++++- src/index.ts | 1 + test/agentTester.test.ts | 22 ++++++- ...tions_runs_4KBSM000000003F4AQ_details.json | 16 ++++- yarn.lock | 12 ++++ 6 files changed, 110 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 2ee78ea..0ec04ad 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "@salesforce/core": "^8.8.0", "@salesforce/kit": "^3.2.3", "@salesforce/sf-plugins-core": "^12.1.0", + "fast-xml-parser": "^4", "nock": "^13.5.6" }, "devDependencies": { diff --git a/src/agentTester.ts b/src/agentTester.ts index 5872e4f..2de0e08 100644 --- a/src/agentTester.ts +++ b/src/agentTester.ts @@ -149,7 +149,7 @@ export class AgentTester { } } -export async function humanFormat(name: string, details: AgentTestDetailsResponse): Promise { +export async function humanFormat(details: AgentTestDetailsResponse): Promise { const { Ux } = await import('@salesforce/sf-plugins-core'); const ux = new Ux(); @@ -176,3 +176,62 @@ export async function humanFormat(name: string, details: AgentTestDetailsRespons export async function jsonFormat(details: AgentTestDetailsResponse): Promise { return Promise.resolve(JSON.stringify(details, null, 2)); } + +export async function junitFormat(details: AgentTestDetailsResponse): Promise { + // Ideally, these would come from the API response. + // Worst case scenario, we cache these values when the customer starts the test run. + // Caching would generally work BUT it's problematic because it doesn't allow the customer to get the results from a test they didn't start on their machine + // and it doesn't allow them to get the results after the TTL cache expires. + const subjectName = 'Copilot_for_Salesforce'; + const testSetName = 'CRM_Sanity_v1'; + + const { XMLBuilder } = await import('fast-xml-parser'); + const builder = new XMLBuilder({ + format: true, + attributeNamePrefix: '$', + ignoreAttributes: false, + }); + + const testCount = details.testCases.length; + const failureCount = details.testCases.filter((tc) => tc.status === 'ERROR').length; + const time = details.testCases.reduce((acc, tc) => { + if (tc.endTime && tc.startTime) { + return acc + new Date(tc.endTime).getTime() - new Date(tc.startTime).getTime(); + } + return acc; + }, 0); + + const suites = builder.build({ + testsuites: { + $name: subjectName, + $tests: testCount, + $failures: failureCount, + $time: time, + property: [ + { $name: 'status', $value: details.status }, + { $name: 'start-time', $value: details.startTime }, + { $name: 'end-time', $value: details.endTime }, + ], + testsuite: details.testCases.map((testCase) => { + const testCaseTime = testCase.endTime + ? new Date(testCase.endTime).getTime() - new Date(testCase.startTime).getTime() + : 0; + + return { + $name: `${testSetName}.${testCase.number}`, + $time: testCaseTime, + $assertions: testCase.expectationResults.length, + failure: testCase.expectationResults + .map((r) => { + if (r.result === 'Failed') { + return { $message: r.errorMessage ?? 'Unknown error' }; + } + }) + .filter((f) => f), + }; + }), + }, + }) as string; + + return `\n${suites}`.trim(); +} diff --git a/src/index.ts b/src/index.ts index 3be5846..bf9e9f3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,7 @@ export { AgentTester, humanFormat, jsonFormat, + junitFormat, type AgentTestDetailsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, diff --git a/test/agentTester.test.ts b/test/agentTester.test.ts index 704b66a..29be8bb 100644 --- a/test/agentTester.test.ts +++ b/test/agentTester.test.ts @@ -4,10 +4,11 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { readFile } from 'node:fs/promises'; import { expect } from 'chai'; import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup'; import { Connection } from '@salesforce/core'; -import { AgentTester } from '../src/agentTester'; +import { AgentTestDetailsResponse, AgentTester, junitFormat } from '../src/agentTester'; describe('AgentTester', () => { const $$ = new TestContext(); @@ -80,3 +81,22 @@ describe('AgentTester', () => { }); }); }); + +describe('junitFormatter', () => { + it('should transform test results to JUnit format', async () => { + const raw = await readFile('./test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json', 'utf8'); + const input = JSON.parse(raw) as AgentTestDetailsResponse; + const output = await junitFormat(input); + expect(output).to.deep.equal(` + + + + + + + + + +`); + }); +}); diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json index b895af0..b025d94 100644 --- a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json +++ b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json @@ -74,7 +74,21 @@ "startTime": "2024-11-28T12:00:32Z", "endTime": "2024-11-28T12:00:33Z", "errorCode": null, - "errorMessage": null + "errorMessage": "Expected \"Result D\" but got \"Result C\"." + }, + { + "name": "topic_sequence_match", + "actualValue": "Result C", + "expectedValue": "Result D", + "score": 0.5, + "result": "Failed", + "metricLabel": "Accuracy", + "metricExplainability": "Measures the correctness of the result.", + "status": "Completed", + "startTime": "2024-11-28T12:00:32Z", + "endTime": "2024-11-28T12:00:33Z", + "errorCode": null, + "errorMessage": "Expected \"Result D\" but got \"Result C\"." } ] } diff --git a/yarn.lock b/yarn.lock index 39f593d..71faa22 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2481,6 +2481,13 @@ fast-uri@^3.0.1: resolved "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz" integrity sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw== +fast-xml-parser@^4: + version "4.5.0" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37" + integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg== + dependencies: + strnum "^1.0.5" + fastest-levenshtein@^1.0.7: version "1.0.16" resolved "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz" @@ -5168,6 +5175,11 @@ strip-json-comments@^3.1.1: resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== +strnum@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db" + integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA== + supports-color@^5.3.0: version "5.5.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz"