feat: support TAP format for test results

forcedotcom · Dec 13, 2024 · f48cb12 · f48cb12
1 parent 0d8fca0
commit f48cb12
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 10 deletions.
diff --git a/src/agentTester.ts b/src/agentTester.ts
@@ -56,6 +56,8 @@ export type AgentTestDetailsResponse = {
   startTime: string;
   endTime?: string;
   errorMessage?: string;
+  subjectName: string;
+  testSetName: string;
   testCases: TestCaseResult[];
 };
 
@@ -178,13 +180,6 @@ export async function jsonFormat(details: AgentTestDetailsResponse): Promise<str
 }
 
 export async function junitFormat(details: AgentTestDetailsResponse): Promise<string> {
-  // Ideally, these would come from the API response.
-  // Worst case scenario, we cache these values when the customer starts the test run.
-  // Caching would generally work BUT it's problematic because it doesn't allow the customer to get the results from a test they didn't start on their machine
-  // and it doesn't allow them to get the results after the TTL cache expires.
-  const subjectName = 'Copilot_for_Salesforce';
-  const testSetName = 'CRM_Sanity_v1';
-
   const { XMLBuilder } = await import('fast-xml-parser');
   const builder = new XMLBuilder({
     format: true,
@@ -203,7 +198,7 @@ export async function junitFormat(details: AgentTestDetailsResponse): Promise<st
 
   const suites = builder.build({
     testsuites: {
-      $name: subjectName,
+      $name: details.subjectName,
       $tests: testCount,
       $failures: failureCount,
       $time: time,
@@ -218,7 +213,7 @@ export async function junitFormat(details: AgentTestDetailsResponse): Promise<st
           : 0;
 
         return {
-          $name: `${testSetName}.${testCase.number}`,
+          $name: `${details.testSetName}.${testCase.number}`,
           $time: testCaseTime,
           $assertions: testCase.expectationResults.length,
           failure: testCase.expectationResults
@@ -235,3 +230,25 @@ export async function junitFormat(details: AgentTestDetailsResponse): Promise<st
 
   return `<?xml version="1.0" encoding="UTF-8"?>\n${suites}`.trim();
 }
+
+export async function tapFormat(details: AgentTestDetailsResponse): Promise<string> {
+  const lines: string[] = [];
+  let expectationCount = 0;
+  for (const testCase of details.testCases) {
+    for (const result of testCase.expectationResults) {
+      const status = result.result === 'Passed' ? 'ok' : 'not ok';
+      expectationCount++;
+      lines.push(`${status} ${expectationCount} ${details.testSetName}.${testCase.number}`);
+      if (status === 'not ok') {
+        lines.push('  ---');
+        lines.push(`  message: ${result.errorMessage ?? 'Unknown error'}`);
+        lines.push(`  expectation: ${result.name}`);
+        lines.push(`  actual: ${result.actualValue}`);
+        lines.push(`  expected: ${result.expectedValue}`);
+        lines.push('  ...');
+      }
+    }
+  }
+
+  return Promise.resolve(`Tap Version 14\n1..${expectationCount}\n${lines.join('\n')}`);
+}
diff --git a/src/index.ts b/src/index.ts
@@ -19,6 +19,7 @@ export {
   humanFormat,
   jsonFormat,
   junitFormat,
+  tapFormat,
   type AgentTestDetailsResponse,
   type AgentTestStartResponse,
   type AgentTestStatusResponse,

diff --git a/test/agentTester.test.ts b/test/agentTester.test.ts
@@ -8,7 +8,7 @@ import { readFile } from 'node:fs/promises';
 import { expect } from 'chai';
 import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup';
 import { Connection } from '@salesforce/core';
-import { AgentTestDetailsResponse, AgentTester, junitFormat } from '../src/agentTester';
+import { AgentTestDetailsResponse, AgentTester, junitFormat, tapFormat } from '../src/agentTester';
 
 describe('AgentTester', () => {
   const $$ = new TestContext();
@@ -100,3 +100,29 @@ describe('junitFormatter', () => {
 </testsuites>`);
   });
 });
+
+describe('tapFormatter', () => {
+  it('should transform test results to TAP format', async () => {
+    const raw = await readFile('./test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json', 'utf8');
+    const input = JSON.parse(raw) as AgentTestDetailsResponse;
+    const output = await tapFormat(input);
+    expect(output).to.deep.equal(`Tap Version 14
+1..4
+ok 1 CRM_Sanity_v1.1
+ok 2 CRM_Sanity_v1.1
+not ok 3 CRM_Sanity_v1.2
+  ---
+  message: Expected "Result D" but got "Result C".
+  expectation: topic_sequence_match
+  actual: Result C
+  expected: Result D
+  ...
+not ok 4 CRM_Sanity_v1.2
+  ---
+  message: Expected "Result D" but got "Result C".
+  expectation: topic_sequence_match
+  actual: Result C
+  expected: Result D
+  ...`);
+  });
+});
diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json
@@ -3,6 +3,8 @@
   "startTime": "2024-11-28T12:00:00Z",
   "endTime": "2024-11-28T12:05:00Z",
   "errorMessage": null,
+  "subjectName": "Copilot_for_Salesforce",
+  "testSetName": "CRM_Sanity_v1",
   "testCases": [
     {
       "status": "COMPLETED",