feat: support junit test format

package.json

@@ -15,6 +15,7 @@
     "@salesforce/core": "^8.8.0",
     "@salesforce/kit": "^3.2.3",
     "@salesforce/sf-plugins-core": "^12.1.0",
+    "fast-xml-parser": "^4",
     "nock": "^13.5.6"
   },
   "devDependencies": {

src/agentTester.ts

@@ -149,7 +149,7 @@ export class AgentTester {
   }
 }
 
-export async function humanFormat(name: string, details: AgentTestDetailsResponse): Promise<string> {
+export async function humanFormat(details: AgentTestDetailsResponse): Promise<string> {
   const { Ux } = await import('@salesforce/sf-plugins-core');
   const ux = new Ux();
 
@@ -176,3 +176,62 @@ export async function humanFormat(name: string, details: AgentTestDetailsRespons
 export async function jsonFormat(details: AgentTestDetailsResponse): Promise<string> {
   return Promise.resolve(JSON.stringify(details, null, 2));
 }
+
+export async function junitFormat(details: AgentTestDetailsResponse): Promise<string> {
+  // Ideally, these would come from the API response.
+  // Worst case scenario, we cache these values when the customer starts the test run.
+  // Caching would generally work BUT it's problematic because it doesn't allow the customer to get the results from a test they didn't start on their machine
+  // and it doesn't allow them to get the results after the TTL cache expires.
+  const subjectName = 'Copilot_for_Salesforce';
+  const testSetName = 'CRM_Sanity_v1';
+
+  const { XMLBuilder } = await import('fast-xml-parser');
+  const builder = new XMLBuilder({
+    format: true,
+    attributeNamePrefix: '$',
+    ignoreAttributes: false,
+  });
+
+  const testCount = details.testCases.length;
+  const failureCount = details.testCases.filter((tc) => tc.status === 'ERROR').length;
+  const time = details.testCases.reduce((acc, tc) => {
+    if (tc.endTime && tc.startTime) {
+      return acc + new Date(tc.endTime).getTime() - new Date(tc.startTime).getTime();
+    }
+    return acc;
+  }, 0);
+
+  const suites = builder.build({
+    testsuites: {
+      $name: subjectName,
+      $tests: testCount,
+      $failures: failureCount,
+      $time: time,
+      property: [
+        { $name: 'status', $value: details.status },
+        { $name: 'start-time', $value: details.startTime },
+        { $name: 'end-time', $value: details.endTime },
+      ],
+      testsuite: details.testCases.map((testCase) => {
+        const testCaseTime = testCase.endTime
+          ? new Date(testCase.endTime).getTime() - new Date(testCase.startTime).getTime()
+          : 0;
+
+        return {
+          $name: `${testSetName}.${testCase.number}`,
+          $time: testCaseTime,
+          $assertions: testCase.expectationResults.length,
+          failure: testCase.expectationResults
+            .map((r) => {
+              if (r.result === 'Failed') {
+                return { $message: r.errorMessage ?? 'Unknown error' };
+              }
+            })
+            .filter((f) => f),
+        };
+      }),
+    },
+  }) as string;
+
+  return `<?xml version="1.0" encoding="UTF-8"?>\n${suites}`.trim();
+}

src/index.ts

@@ -18,6 +18,7 @@ export {
   AgentTester,
   humanFormat,
   jsonFormat,
+  junitFormat,
   type AgentTestDetailsResponse,
   type AgentTestStartResponse,
   type AgentTestStatusResponse,

test/agentTester.test.ts

@@ -4,10 +4,11 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
+import { readFile } from 'node:fs/promises';
 import { expect } from 'chai';
 import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup';
 import { Connection } from '@salesforce/core';
-import { AgentTester } from '../src/agentTester';
+import { AgentTestDetailsResponse, AgentTester, junitFormat } from '../src/agentTester';
 
 describe('AgentTester', () => {
   const $$ = new TestContext();
@@ -80,3 +81,22 @@ describe('AgentTester', () => {
     });
   });
 });
+
+describe('junitFormatter', () => {
+  it('should transform test results to JUnit format', async () => {
+    const raw = await readFile('./test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json', 'utf8');
+    const input = JSON.parse(raw) as AgentTestDetailsResponse;
+    const output = await junitFormat(input);
+    expect(output).to.deep.equal(`<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="Copilot_for_Salesforce" tests="2" failures="1" time="20000">
+  <property name="status" value="COMPLETED"></property>
+  <property name="start-time" value="2024-11-28T12:00:00Z"></property>
+  <property name="end-time" value="2024-11-28T12:05:00Z"></property>
+  <testsuite name="CRM_Sanity_v1.1" time="10000" assertions="2"></testsuite>
+  <testsuite name="CRM_Sanity_v1.2" time="10000" assertions="2">
+    <failure message="Expected &quot;Result D&quot; but got &quot;Result C&quot;."></failure>
+    <failure message="Expected &quot;Result D&quot; but got &quot;Result C&quot;."></failure>
+  </testsuite>
+</testsuites>`);
+  });
+});

test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json

@@ -74,7 +74,21 @@
           "startTime": "2024-11-28T12:00:32Z",
           "endTime": "2024-11-28T12:00:33Z",
           "errorCode": null,
-          "errorMessage": null
+          "errorMessage": "Expected \"Result D\" but got \"Result C\"."
+        },
+        {
+          "name": "topic_sequence_match",
+          "actualValue": "Result C",
+          "expectedValue": "Result D",
+          "score": 0.5,
+          "result": "Failed",
+          "metricLabel": "Accuracy",
+          "metricExplainability": "Measures the correctness of the result.",
+          "status": "Completed",
+          "startTime": "2024-11-28T12:00:32Z",
+          "endTime": "2024-11-28T12:00:33Z",
+          "errorCode": null,
+          "errorMessage": "Expected \"Result D\" but got \"Result C\"."
         }
       ]
     }

yarn.lock

@@ -2441,6 +2441,13 @@ fast-uri@^3.0.1:
   resolved "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz"
   integrity sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==
 
+fast-xml-parser@^4:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37"
+  integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==
+  dependencies:
+    strnum "^1.0.5"
+
 fastest-levenshtein@^1.0.7:
   version "1.0.16"
   resolved "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz"
@@ -5144,6 +5151,11 @@ strip-json-comments@^3.1.1:
   resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz"
   integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
 
+strnum@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db"
+  integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==
+
 supports-color@^5.3.0:
   version "5.5.0"
   resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz"