feat: support TAP format for test results

.github/workflows/create-github-release.yml

@@ -6,12 +6,12 @@ on:
       - main
       - prerelease/**
     tags-ignore:
-      - "*"
+      - '*'
   workflow_dispatch:
     inputs:
       prerelease:
         type: string
-        description: "Name to use for the prerelease: beta, dev, etc. NOTE: If this is already set in the package.json, it does not need to be passed in here."
+        description: 'Name to use for the prerelease: beta, dev, etc. NOTE: If this is already set in the package.json, it does not need to be passed in here.'
 
 jobs:
   release:

CHANGELOG.md

@@ -1,182 +1,122 @@
 ## [0.4.5](https://github.com/forcedotcom/agents/compare/0.4.4...0.4.5) (2024-12-14)
 
-
 ### Bug Fixes
 
-* **deps:** bump @salesforce/source-deploy-retrieve ([a282749](https://github.com/forcedotcom/agents/commit/a28274962414ee576e9da3e3ba8df21eab3ab9ad))
-
-
+- **deps:** bump @salesforce/source-deploy-retrieve ([a282749](https://github.com/forcedotcom/agents/commit/a28274962414ee576e9da3e3ba8df21eab3ab9ad))
 
 ## [0.4.4](https://github.com/forcedotcom/agents/compare/0.4.3...0.4.4) (2024-12-13)
 
-
-
 ## [0.4.3](https://github.com/forcedotcom/agents/compare/0.4.2...0.4.3) (2024-12-13)
 
-
-
 ## [0.4.2](https://github.com/forcedotcom/agents/compare/0.4.1...0.4.2) (2024-12-10)
 
-
 ### Bug Fixes
 
-* retrieve GenAiPlugins too ([1d461a2](https://github.com/forcedotcom/agents/commit/1d461a27a713212bb2f033046a71d47d3be21204))
-
-
+- retrieve GenAiPlugins too ([1d461a2](https://github.com/forcedotcom/agents/commit/1d461a27a713212bb2f033046a71d47d3be21204))
 
 ## [0.4.1](https://github.com/forcedotcom/agents/compare/0.4.0...0.4.1) (2024-12-10)
 
-
 ### Bug Fixes
 
-* agent create working ([1a57a90](https://github.com/forcedotcom/agents/commit/1a57a90630a043ad6294a9895145fa4fd7442330))
-* agent.create WIP ([836f6db](https://github.com/forcedotcom/agents/commit/836f6db06cd529d0482b9fa9630b9314892b799f))
-* emit lifecycle events ([c3bba31](https://github.com/forcedotcom/agents/commit/c3bba3100f953a320efb4542f0ecc031a5d41179))
-
-
+- agent create working ([1a57a90](https://github.com/forcedotcom/agents/commit/1a57a90630a043ad6294a9895145fa4fd7442330))
+- agent.create WIP ([836f6db](https://github.com/forcedotcom/agents/commit/836f6db06cd529d0482b9fa9630b9314892b799f))
+- emit lifecycle events ([c3bba31](https://github.com/forcedotcom/agents/commit/c3bba3100f953a320efb4542f0ecc031a5d41179))
 
 # [0.4.0](https://github.com/forcedotcom/agents/compare/0.3.1...0.4.0) (2024-12-10)
 
-
 ### Features
 
-* junit result formatter ([42ff64b](https://github.com/forcedotcom/agents/commit/42ff64bd855d4de5e4f6585ab5c141816d1bccf3))
-
-
+- junit result formatter ([42ff64b](https://github.com/forcedotcom/agents/commit/42ff64bd855d4de5e4f6585ab5c141816d1bccf3))
 
 ## [0.3.1](https://github.com/forcedotcom/agents/compare/0.3.0...0.3.1) (2024-12-07)
 
-
 ### Bug Fixes
 
-* **deps:** bump @oclif/table from 0.3.3 to 0.3.5 ([49ce6d5](https://github.com/forcedotcom/agents/commit/49ce6d58fe3a433a14bad20d38240180aaff864f))
-
-
+- **deps:** bump @oclif/table from 0.3.3 to 0.3.5 ([49ce6d5](https://github.com/forcedotcom/agents/commit/49ce6d58fe3a433a14bad20d38240180aaff864f))
 
 # [0.3.0](https://github.com/forcedotcom/agents/compare/0.2.4...0.3.0) (2024-12-05)
 
-
 ### Features
 
-* export test formatters ([b5f26df](https://github.com/forcedotcom/agents/commit/b5f26df835c3a08e9065f327afc4a6bc1997c9ab))
-
-
+- export test formatters ([b5f26df](https://github.com/forcedotcom/agents/commit/b5f26df835c3a08e9065f327afc4a6bc1997c9ab))
 
 ## [0.2.4](https://github.com/forcedotcom/agents/compare/0.2.3...0.2.4) (2024-12-03)
 
-
 ### Bug Fixes
 
-* add doc for agent class ([5208345](https://github.com/forcedotcom/agents/commit/52083450e7eff71e1cc3caf570abd18939317d9a))
-
-
+- add doc for agent class ([5208345](https://github.com/forcedotcom/agents/commit/52083450e7eff71e1cc3caf570abd18939317d9a))
 
 ## [0.2.3](https://github.com/forcedotcom/agents/compare/0.2.2...0.2.3) (2024-12-03)
 
-
-
 ## [0.2.2](https://github.com/forcedotcom/agents/compare/0.2.1...0.2.2) (2024-12-02)
 
-
 ### Bug Fixes
 
-* export more types ([0a1d408](https://github.com/forcedotcom/agents/commit/0a1d408dd0d815b36509d5b5fe343e7e0817d1a2))
-
-
+- export more types ([0a1d408](https://github.com/forcedotcom/agents/commit/0a1d408dd0d815b36509d5b5fe343e7e0817d1a2))
 
 ## [0.2.1](https://github.com/forcedotcom/agents/compare/0.2.0...0.2.1) (2024-12-02)
 
-
 ### Bug Fixes
 
-* update return type on start ([c14e8c4](https://github.com/forcedotcom/agents/commit/c14e8c41c3180dad1df807a3531e944b89cce229))
-
-
+- update return type on start ([c14e8c4](https://github.com/forcedotcom/agents/commit/c14e8c41c3180dad1df807a3531e944b89cce229))
 
 # [0.2.0](https://github.com/forcedotcom/agents/compare/0.1.6...0.2.0) (2024-12-02)
 
-
 ### Bug Fixes
 
-* add polling lifecycle events ([695fd08](https://github.com/forcedotcom/agents/commit/695fd086865c60850d53aa2753686ea5aeef2d4a))
-* use sf-plugins-core for making table ([97eaa63](https://github.com/forcedotcom/agents/commit/97eaa633fd739c214029ce0fb1dbd521f220c5ae))
-
+- add polling lifecycle events ([695fd08](https://github.com/forcedotcom/agents/commit/695fd086865c60850d53aa2753686ea5aeef2d4a))
+- use sf-plugins-core for making table ([97eaa63](https://github.com/forcedotcom/agents/commit/97eaa633fd739c214029ce0fb1dbd521f220c5ae))
 
 ### Features
 
-* add cancel method ([8371f9f](https://github.com/forcedotcom/agents/commit/8371f9fd735bfd7e12dd4a95419e321bb34cf465))
-* mock agent testing ([8df61a9](https://github.com/forcedotcom/agents/commit/8df61a9fba8005d0823bba5ce5f14d3ab5a5c12e))
-* mocked agent testing ([334988d](https://github.com/forcedotcom/agents/commit/334988d753f942fbfecdaa776e2285c51b81ebf5))
-* poll both status and details ([61b03dc](https://github.com/forcedotcom/agents/commit/61b03dcba132ed07df194953c850595771f3ccff))
-
-
+- add cancel method ([8371f9f](https://github.com/forcedotcom/agents/commit/8371f9fd735bfd7e12dd4a95419e321bb34cf465))
+- mock agent testing ([8df61a9](https://github.com/forcedotcom/agents/commit/8df61a9fba8005d0823bba5ce5f14d3ab5a5c12e))
+- mocked agent testing ([334988d](https://github.com/forcedotcom/agents/commit/334988d753f942fbfecdaa776e2285c51b81ebf5))
+- poll both status and details ([61b03dc](https://github.com/forcedotcom/agents/commit/61b03dcba132ed07df194953c850595771f3ccff))
 
 ## [0.1.6](https://github.com/forcedotcom/agents/compare/0.1.5...0.1.6) (2024-11-16)
 
-
 ### Bug Fixes
 
-* **deps:** bump cross-spawn from 7.0.3 to 7.0.5 ([7f43cc7](https://github.com/forcedotcom/agents/commit/7f43cc706b848fd54c88d04bee2c0b7b632d7e76))
-
-
+- **deps:** bump cross-spawn from 7.0.3 to 7.0.5 ([7f43cc7](https://github.com/forcedotcom/agents/commit/7f43cc706b848fd54c88d04bee2c0b7b632d7e76))
 
 ## [0.1.5](https://github.com/forcedotcom/agents/compare/0.1.4...0.1.5) (2024-11-16)
 
-
 ### Bug Fixes
 
-* **deps:** bump @salesforce/core from 8.6.3 to 8.8.0 ([193237b](https://github.com/forcedotcom/agents/commit/193237b5dbbe7ce1ee596a3b7305b5602d0883f8))
-
-
+- **deps:** bump @salesforce/core from 8.6.3 to 8.8.0 ([193237b](https://github.com/forcedotcom/agents/commit/193237b5dbbe7ce1ee596a3b7305b5602d0883f8))
 
 ## [0.1.4](https://github.com/forcedotcom/agents/compare/0.1.3...0.1.4) (2024-11-12)
 
-
 ### Bug Fixes
 
-* do not append spec in name ([284d5d5](https://github.com/forcedotcom/agents/commit/284d5d56ed99c67b93a65904a00fdb00a2552a0e))
-
-
+- do not append spec in name ([284d5d5](https://github.com/forcedotcom/agents/commit/284d5d56ed99c67b93a65904a00fdb00a2552a0e))
 
 ## [0.1.3](https://github.com/forcedotcom/agents/compare/0.1.2...0.1.3) (2024-11-12)
 
-
 ### Bug Fixes
 
-* use latest ([92ecbba](https://github.com/forcedotcom/agents/commit/92ecbbabc403fe57bf4069f9928b029d23db7a16))
-
-
+- use latest ([92ecbba](https://github.com/forcedotcom/agents/commit/92ecbbabc403fe57bf4069f9928b029d23db7a16))
 
 ## [0.1.2](https://github.com/forcedotcom/agents/compare/0.1.1...0.1.2) (2024-11-12)
 
-
 ### Bug Fixes
 
-* publish to preview ([3f5ccb6](https://github.com/forcedotcom/agents/commit/3f5ccb687017186eb29b8b18c7fdce33daee1f70))
-
-
+- publish to preview ([3f5ccb6](https://github.com/forcedotcom/agents/commit/3f5ccb687017186eb29b8b18c7fdce33daee1f70))
 
 ## [0.1.1](https://github.com/forcedotcom/agents/compare/0.1.0...0.1.1) (2024-11-10)
 
-
 ### Bug Fixes
 
-* export Agent class ([6c42b63](https://github.com/forcedotcom/agents/commit/6c42b63bbe9a5a5cf6fa0cea8f5649d07aaa6adc))
-
-
+- export Agent class ([6c42b63](https://github.com/forcedotcom/agents/commit/6c42b63bbe9a5a5cf6fa0cea8f5649d07aaa6adc))
 
 # [0.1.0](https://github.com/forcedotcom/agents/compare/0c5d8d6ab9e9a8470c7192a56350567882a3017b...0.1.0) (2024-11-09)
 
-
 ### Bug Fixes
 
-* improve types and linting ([d5a6cb3](https://github.com/forcedotcom/agents/commit/d5a6cb3348e63d52e10540e99cf509be64a26649))
-* revise readme and version ([f690b7f](https://github.com/forcedotcom/agents/commit/f690b7f8a911315f467f00f5f533e22e92c69a9e))
-
+- improve types and linting ([d5a6cb3](https://github.com/forcedotcom/agents/commit/d5a6cb3348e63d52e10540e99cf509be64a26649))
+- revise readme and version ([f690b7f](https://github.com/forcedotcom/agents/commit/f690b7f8a911315f467f00f5f533e22e92c69a9e))
 
 ### Features
 
-* add initial agent job spec create and mock ([0c5d8d6](https://github.com/forcedotcom/agents/commit/0c5d8d6ab9e9a8470c7192a56350567882a3017b))
-
-
-
+- add initial agent job spec create and mock ([0c5d8d6](https://github.com/forcedotcom/agents/commit/0c5d8d6ab9e9a8470c7192a56350567882a3017b))

src/agentTester.ts

@@ -56,6 +56,8 @@ export type AgentTestDetailsResponse = {
   startTime: string;
   endTime?: string;
   errorMessage?: string;
+  subjectName: string;
+  testSetName: string;
   testCases: TestCaseResult[];
 };
 
@@ -205,13 +207,6 @@ export async function jsonFormat(details: AgentTestDetailsResponse): Promise<str
 }
 
 export async function junitFormat(details: AgentTestDetailsResponse): Promise<string> {
-  // Ideally, these would come from the API response.
-  // Worst case scenario, we cache these values when the customer starts the test run.
-  // Caching would generally work BUT it's problematic because it doesn't allow the customer to get the results from a test they didn't start on their machine
-  // and it doesn't allow them to get the results after the TTL cache expires.
-  const subjectName = 'Copilot_for_Salesforce';
-  const testSetName = 'CRM_Sanity_v1';
-
   // eslint-disable-next-line import/no-extraneous-dependencies
   const { XMLBuilder } = await import('fast-xml-parser');
   const builder = new XMLBuilder({
@@ -231,7 +226,7 @@ export async function junitFormat(details: AgentTestDetailsResponse): Promise<st
 
   const suites = builder.build({
     testsuites: {
-      $name: subjectName,
+      $name: details.subjectName,
       $tests: testCount,
       $failures: failureCount,
       $time: time,
@@ -246,7 +241,7 @@ export async function junitFormat(details: AgentTestDetailsResponse): Promise<st
           : 0;
 
         return {
-          $name: `${testSetName}.${testCase.number}`,
+          $name: `${details.testSetName}.${testCase.number}`,
           $time: testCaseTime,
           $assertions: testCase.expectationResults.length,
           failure: testCase.expectationResults
@@ -263,3 +258,25 @@ export async function junitFormat(details: AgentTestDetailsResponse): Promise<st
 
   return `<?xml version="1.0" encoding="UTF-8"?>\n${suites}`.trim();
 }
+
+export async function tapFormat(details: AgentTestDetailsResponse): Promise<string> {
+  const lines: string[] = [];
+  let expectationCount = 0;
+  for (const testCase of details.testCases) {
+    for (const result of testCase.expectationResults) {
+      const status = result.result === 'Passed' ? 'ok' : 'not ok';
+      expectationCount++;
+      lines.push(`${status} ${expectationCount} ${details.testSetName}.${testCase.number}`);
+      if (status === 'not ok') {
+        lines.push('  ---');
+        lines.push(`  message: ${result.errorMessage ?? 'Unknown error'}`);
+        lines.push(`  expectation: ${result.name}`);
+        lines.push(`  actual: ${result.actualValue}`);
+        lines.push(`  expected: ${result.expectedValue}`);
+        lines.push('  ...');
+      }
+    }
+  }
+
+  return Promise.resolve(`Tap Version 14\n1..${expectationCount}\n${lines.join('\n')}`);
+}

src/index.ts

@@ -19,6 +19,7 @@ export {
   humanFormat,
   jsonFormat,
   junitFormat,
+  tapFormat,
   type AgentTestDetailsResponse,
   type AgentTestStartResponse,
   type AgentTestStatusResponse,

test/agentTester.test.ts

@@ -8,7 +8,7 @@ import { readFile } from 'node:fs/promises';
 import { expect } from 'chai';
 import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup';
 import { Connection } from '@salesforce/core';
-import { AgentTestDetailsResponse, AgentTester, junitFormat } from '../src/agentTester';
+import { AgentTestDetailsResponse, AgentTester, junitFormat, tapFormat } from '../src/agentTester';
 
 describe('AgentTester', () => {
   const $$ = new TestContext();
@@ -100,3 +100,29 @@ describe('junitFormatter', () => {
 </testsuites>`);
   });
 });
+
+describe('tapFormatter', () => {
+  it('should transform test results to TAP format', async () => {
+    const raw = await readFile('./test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json', 'utf8');
+    const input = JSON.parse(raw) as AgentTestDetailsResponse;
+    const output = await tapFormat(input);
+    expect(output).to.deep.equal(`Tap Version 14
+1..4
+ok 1 CRM_Sanity_v1.1
+ok 2 CRM_Sanity_v1.1
+not ok 3 CRM_Sanity_v1.2
+  ---
+  message: Expected "Result D" but got "Result C".
+  expectation: topic_sequence_match
+  actual: Result C
+  expected: Result D
+  ...
+not ok 4 CRM_Sanity_v1.2
+  ---
+  message: Expected "Result D" but got "Result C".
+  expectation: topic_sequence_match
+  actual: Result C
+  expected: Result D
+  ...`);
+  });
+});

test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json

@@ -3,6 +3,8 @@
   "startTime": "2024-11-28T12:00:00Z",
   "endTime": "2024-11-28T12:05:00Z",
   "errorMessage": null,
+  "subjectName": "Copilot_for_Salesforce",
+  "testSetName": "CRM_Sanity_v1",
   "testCases": [
     {
       "status": "COMPLETED",