feat: implement agent test run

command-snapshot.json

@@ -39,8 +39,8 @@
     "alias": [],
     "command": "agent:test:run",
     "flagAliases": [],
-    "flagChars": ["d", "i", "o", "w"],
-    "flags": ["flags-dir", "id", "json", "output-dir", "target-org", "wait"],
+    "flagChars": ["d", "i", "o", "r", "w"],
+    "flags": ["api-version", "flags-dir", "id", "json", "output-dir", "result-format", "target-org", "wait"],
     "plugin": "@salesforce/plugin-agent"
   }
 ]

messages/agent.test.run.md

@@ -26,6 +26,10 @@ If the command continues to run after the wait period, the CLI returns control o
 
 Directory in which to store test run files.
 
+# flags.result-format.summary
+
+Format of the test run results.
+
 # examples
 
 - Start a test for an Agent:

package.json

@@ -9,15 +9,16 @@
     "@inquirer/input": "^4.0.1",
     "@inquirer/select": "^4.0.1",
     "@oclif/core": "^4",
-    "@salesforce/agents": "^0.1.4",
     "@oclif/multi-stage-output": "^0.7.12",
+    "@salesforce/agents": "^0.1.4",
     "@salesforce/core": "^8.5.2",
     "@salesforce/kit": "^3.2.1",
     "@salesforce/sf-plugins-core": "^12",
     "ansis": "^3.3.2"
   },
   "devDependencies": {
     "@oclif/plugin-command-snapshot": "^5.2.19",
+    "@oclif/test": "^4.1.0",
     "@salesforce/cli-plugins-testkit": "^5.3.35",
     "@salesforce/dev-scripts": "^10.2.10",
     "@salesforce/plugin-command-reference": "^3.1.29",

src/commands/agent/test/run.ts

@@ -5,12 +5,18 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
+import { MultiStageOutput } from '@oclif/multi-stage-output';
 import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
-import { Messages } from '@salesforce/core';
+import { Lifecycle, Messages } from '@salesforce/core';
+import { AgentTester } from '@salesforce/agents';
+import { colorize } from '@oclif/core/ux';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
 const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run');
 
+const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } =>
+  (e as { name: string })?.name === 'PollingClientTimeout';
+
 export type AgentTestRunResult = {
   jobId: string; // AiEvaluation.Id
   success: boolean;
@@ -25,13 +31,16 @@ export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
 
   public static readonly flags = {
     'target-org': Flags.requiredOrg(),
+    'api-version': Flags.orgApiVersion(),
     // AiEvalDefinitionVersion.Id -- This should really be "test-name"
     id: Flags.string({
       char: 'i',
       required: true,
       summary: messages.getMessage('flags.id.summary'),
       description: messages.getMessage('flags.id.description'),
     }),
+    // we want to pass `undefined` to the API
+    // eslint-disable-next-line sf-plugin/flag-min-max-default
     wait: Flags.duration({
       char: 'w',
       unit: 'minutes',
@@ -43,27 +52,73 @@ export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
       char: 'd',
       summary: messages.getMessage('flags.output-dir.summary'),
     }),
+    'result-format': Flags.option({
+      options: ['json', 'human', 'tap', 'junit'],
+      default: 'human',
+      char: 'r',
+      summary: messages.getMessage('flags.result-format.summary'),
+    })(),
     //
     // Future flags:
-    //   result-format [csv, json, table, junit, TAP]
     //   suites [array of suite names]
     //   verbose [boolean]
-    //   ??? api-version or build-version ???
   };
 
   public async run(): Promise<AgentTestRunResult> {
     const { flags } = await this.parse(AgentTestRun);
+    const mso = new MultiStageOutput<{ id: string; status: string }>({
+      jsonEnabled: this.jsonEnabled(),
+      title: `Agent Test Run: ${flags.id}`,
+      stages: ['Starting Tests', 'Polling for Test Results'],
+      stageSpecificBlock: [
+        {
+          stage: 'Polling for Test Results',
+          type: 'dynamic-key-value',
+          label: 'Status',
+          get: (data) => data?.status,
+        },
+      ],
+      postStagesBlock: [
+        {
+          type: 'dynamic-key-value',
+          label: 'Job ID',
+          get: (data) => data?.id,
+        },
+      ],
+    });
+    mso.skipTo('Starting Tests');
+    const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version']));
+    const response = await agentTester.start(flags.id);
+    mso.updateData({ id: response.id });
+    if (flags.wait?.minutes) {
+      mso.skipTo('Polling for Test Results');
+      const lifecycle = Lifecycle.getInstance();
+      lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) =>
+        Promise.resolve(mso.updateData({ status: event?.status }))
+      );
+      try {
+        const { formatted } = await agentTester.poll(response.id, { timeout: flags.wait });
+        mso.stop();
+        this.log(formatted);
+      } catch (e) {
+        if (isTimeoutError(e)) {
+          mso.stop('async');
+          this.log(`Client timed out after ${flags.wait.minutes} minutes.`);
+          this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`);
+        } else {
+          mso.error();
+          throw e;
+        }
+      }
+    } else {
+      mso.stop();
+      this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`);
+    }
 
-    this.log(`Starting tests for AiEvalDefinitionVersion: ${flags.id}`);
-
-    // Call SF Eval Connect API passing AiEvalDefinitionVersion.Id
-    // POST to /einstein/ai-evaluations/{aiEvalDefinitionVersionId}/start
-
-    // Returns: AiEvaluation.Id
-
+    mso.stop();
     return {
       success: true,
-      jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id; needed for getting status and stopping
+      jobId: response.id, // AiEvaluation.Id; needed for getting status and stopping
     };
   }
 }

test/unit/agent-test-run.test.ts

@@ -4,30 +4,21 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
+
+import { runCommand } from '@oclif/test';
 import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup';
 import { expect } from 'chai';
-import { stubSfCommandUx } from '@salesforce/sf-plugins-core';
-import AgentTestRun from '../../src/commands/agent/test/run.js';
 
 describe('agent run test', () => {
   const $$ = new TestContext();
   const testOrg = new MockTestOrgData();
-  let sfCommandStubs: ReturnType<typeof stubSfCommandUx>;
-
-  beforeEach(() => {
-    sfCommandStubs = stubSfCommandUx($$.SANDBOX);
-  });
 
   afterEach(() => {
     $$.restore();
   });
 
   it('runs agent run test', async () => {
-    await AgentTestRun.run(['-i', 'the-id', '-o', testOrg.username]);
-    const output = sfCommandStubs.log
-      .getCalls()
-      .flatMap((c) => c.args)
-      .join('\n');
-    expect(output).to.include('Starting tests for AiEvalDefinitionVersion:');
+    const { stdout } = await runCommand(`agent:test:run -i the-id -o ${testOrg.username}`);
+    expect(stdout).to.include('Agent Test Run: the-id');
   });
 });

yarn.lock

@@ -1365,6 +1365,14 @@
     strip-ansi "^7.1.0"
     wrap-ansi "^9.0.0"
 
+"@oclif/test@^4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@oclif/test/-/test-4.1.0.tgz#7935e3707cf07480790139e02973196d18d16822"
+  integrity sha512-2ugir6NhRsWJqHM9d2lMEWNiOTD678Jlx5chF/fg6TCAlc7E6E/6+zt+polrCTnTIpih5P/HxOtDekgtjgARwQ==
+  dependencies:
+    ansis "^3.3.2"
+    debug "^4.3.6"
+
 "@pkgjs/parseargs@^0.11.0":
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
@@ -3217,7 +3225,7 @@ dateformat@^4.6.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
   integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
 
-debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.7:
+debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.6, debug@^4.3.7:
   version "4.3.7"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52"
   integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==