From 81ec3a3d3fb253e8d721881dfba7fb0036323e84 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Thu, 14 Nov 2024 13:31:41 -0700 Subject: [PATCH] feat: implement agent test run --- command-snapshot.json | 4 +- messages/agent.test.run.md | 4 ++ package.json | 3 +- src/commands/agent/test/run.ts | 77 +++++++++++++++++++++++++++----- test/unit/agent-test-run.test.ts | 17 ++----- yarn.lock | 10 ++++- 6 files changed, 87 insertions(+), 28 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index 176d7c7..0dba124 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -39,8 +39,8 @@ "alias": [], "command": "agent:test:run", "flagAliases": [], - "flagChars": ["d", "i", "o", "w"], - "flags": ["flags-dir", "id", "json", "output-dir", "target-org", "wait"], + "flagChars": ["d", "i", "o", "r", "w"], + "flags": ["api-version", "flags-dir", "id", "json", "output-dir", "result-format", "target-org", "wait"], "plugin": "@salesforce/plugin-agent" } ] diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 4b73b25..0d5a90a 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -26,6 +26,10 @@ If the command continues to run after the wait period, the CLI returns control o Directory in which to store test run files. +# flags.result-format.summary + +Format of the test run results. + # examples - Start a test for an Agent: diff --git a/package.json b/package.json index 6e40d2f..2824bb6 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,8 @@ "@inquirer/input": "^4.0.1", "@inquirer/select": "^4.0.1", "@oclif/core": "^4", - "@salesforce/agents": "^0.1.4", "@oclif/multi-stage-output": "^0.7.12", + "@salesforce/agents": "^0.1.4", "@salesforce/core": "^8.5.2", "@salesforce/kit": "^3.2.1", "@salesforce/sf-plugins-core": "^12", @@ -18,6 +18,7 @@ }, "devDependencies": { "@oclif/plugin-command-snapshot": "^5.2.19", + "@oclif/test": "^4.1.0", "@salesforce/cli-plugins-testkit": "^5.3.35", "@salesforce/dev-scripts": "^10.2.10", "@salesforce/plugin-command-reference": "^3.1.29", diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 2a55c7a..45b4269 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -5,12 +5,18 @@ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { MultiStageOutput } from '@oclif/multi-stage-output'; import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { Lifecycle, Messages } from '@salesforce/core'; +import { AgentTester } from '@salesforce/agents'; +import { colorize } from '@oclif/core/ux'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); +const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => + (e as { name: string })?.name === 'PollingClientTimeout'; + export type AgentTestRunResult = { jobId: string; // AiEvaluation.Id success: boolean; @@ -25,6 +31,7 @@ export default class AgentTestRun extends SfCommand { public static readonly flags = { 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), // AiEvalDefinitionVersion.Id -- This should really be "test-name" id: Flags.string({ char: 'i', @@ -32,6 +39,8 @@ export default class AgentTestRun extends SfCommand { summary: messages.getMessage('flags.id.summary'), description: messages.getMessage('flags.id.description'), }), + // we want to pass `undefined` to the API + // eslint-disable-next-line sf-plugin/flag-min-max-default wait: Flags.duration({ char: 'w', unit: 'minutes', @@ -43,27 +52,73 @@ export default class AgentTestRun extends SfCommand { char: 'd', summary: messages.getMessage('flags.output-dir.summary'), }), + 'result-format': Flags.option({ + options: ['json', 'human', 'tap', 'junit'], + default: 'human', + char: 'r', + summary: messages.getMessage('flags.result-format.summary'), + })(), // // Future flags: - // result-format [csv, json, table, junit, TAP] // suites [array of suite names] // verbose [boolean] - // ??? api-version or build-version ??? }; public async run(): Promise { const { flags } = await this.parse(AgentTestRun); + const mso = new MultiStageOutput<{ id: string; status: string }>({ + jsonEnabled: this.jsonEnabled(), + title: `Agent Test Run: ${flags.id}`, + stages: ['Starting Tests', 'Polling for Test Results'], + stageSpecificBlock: [ + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Status', + get: (data) => data?.status, + }, + ], + postStagesBlock: [ + { + type: 'dynamic-key-value', + label: 'Job ID', + get: (data) => data?.id, + }, + ], + }); + mso.skipTo('Starting Tests'); + const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + const response = await agentTester.start(flags.id); + mso.updateData({ id: response.id }); + if (flags.wait?.minutes) { + mso.skipTo('Polling for Test Results'); + const lifecycle = Lifecycle.getInstance(); + lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => + Promise.resolve(mso.updateData({ status: event?.status })) + ); + try { + const { formatted } = await agentTester.poll(response.id, { timeout: flags.wait }); + mso.stop(); + this.log(formatted); + } catch (e) { + if (isTimeoutError(e)) { + mso.stop('async'); + this.log(`Client timed out after ${flags.wait.minutes} minutes.`); + this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`); + } else { + mso.error(); + throw e; + } + } + } else { + mso.stop(); + this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`); + } - this.log(`Starting tests for AiEvalDefinitionVersion: ${flags.id}`); - - // Call SF Eval Connect API passing AiEvalDefinitionVersion.Id - // POST to /einstein/ai-evaluations/{aiEvalDefinitionVersionId}/start - - // Returns: AiEvaluation.Id - + mso.stop(); return { success: true, - jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id; needed for getting status and stopping + jobId: response.id, // AiEvaluation.Id; needed for getting status and stopping }; } } diff --git a/test/unit/agent-test-run.test.ts b/test/unit/agent-test-run.test.ts index 0bb1cd1..0b13124 100644 --- a/test/unit/agent-test-run.test.ts +++ b/test/unit/agent-test-run.test.ts @@ -4,30 +4,21 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ + +import { runCommand } from '@oclif/test'; import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup'; import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; -import AgentTestRun from '../../src/commands/agent/test/run.js'; describe('agent run test', () => { const $$ = new TestContext(); const testOrg = new MockTestOrgData(); - let sfCommandStubs: ReturnType; - - beforeEach(() => { - sfCommandStubs = stubSfCommandUx($$.SANDBOX); - }); afterEach(() => { $$.restore(); }); it('runs agent run test', async () => { - await AgentTestRun.run(['-i', 'the-id', '-o', testOrg.username]); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('Starting tests for AiEvalDefinitionVersion:'); + const { stdout } = await runCommand(`agent:test:run -i the-id -o ${testOrg.username}`); + expect(stdout).to.include('Agent Test Run: the-id'); }); }); diff --git a/yarn.lock b/yarn.lock index 923425d..0588b29 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1365,6 +1365,14 @@ strip-ansi "^7.1.0" wrap-ansi "^9.0.0" +"@oclif/test@^4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@oclif/test/-/test-4.1.0.tgz#7935e3707cf07480790139e02973196d18d16822" + integrity sha512-2ugir6NhRsWJqHM9d2lMEWNiOTD678Jlx5chF/fg6TCAlc7E6E/6+zt+polrCTnTIpih5P/HxOtDekgtjgARwQ== + dependencies: + ansis "^3.3.2" + debug "^4.3.6" + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" @@ -3217,7 +3225,7 @@ dateformat@^4.6.3: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.7: +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.6, debug@^4.3.7: version "4.3.7" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==