feat: mock agent tests

package.json

@@ -11,8 +11,10 @@
     "url": "https://github.com/forcedotcom/agents.git"
   },
   "dependencies": {
+    "@oclif/table": "^0.3.3",
     "@salesforce/core": "^8.5.2",
-    "@salesforce/kit": "^3.2.3"
+    "@salesforce/kit": "^3.2.3",
+    "nock": "^13.5.6"
   },
   "devDependencies": {
     "@salesforce/cli-plugins-testkit": "^5.3.20",

src/agent.ts

@@ -5,12 +5,10 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-import { join } from 'node:path';
-import { readFileSync, statSync } from 'node:fs';
 import { inspect } from 'node:util';
 import { Connection, Logger, SfError, SfProject } from '@salesforce/core';
 import { Duration, sleep } from '@salesforce/kit';
-import { getMockDir } from './mockDir';
+import { MaybeMock } from './maybe-mock';
 import {
   type SfAgent,
   type AgentCreateConfig,
@@ -22,11 +20,11 @@ import {
 
 export class Agent implements SfAgent {
   private logger: Logger;
-  private mockDir?: string;
+  private maybeMock: MaybeMock;
 
-  public constructor(private connection: Connection, private project: SfProject) {
+  public constructor(connection: Connection, private project: SfProject) {
     this.logger = Logger.childFromRoot(this.constructor.name);
-    this.mockDir = getMockDir();
+    this.maybeMock = new MaybeMock(connection);
   }
 
   public async create(config: AgentCreateConfig): Promise<AgentCreateResponse> {
@@ -50,48 +48,14 @@ export class Agent implements SfAgent {
     this.verifyAgentSpecConfig(config);
 
     let agentSpec: AgentJobSpec;
-
-    if (this.mockDir) {
-      const specFileName = `${config.name}.json`;
-      const specFilePath = join(this.mockDir, `${specFileName}`);
-      try {
-        this.logger.debug(`Using mock directory: ${this.mockDir} for agent job spec creation`);
-        statSync(specFilePath);
-      } catch (err) {
-        throw SfError.create({
-          name: 'MissingMockFile',
-          message: `SF_MOCK_DIR [${this.mockDir}] must contain a spec file with name ${specFileName}`,
-          cause: err,
-        });
-      }
-      try {
-        this.logger.debug(`Returning mock agent spec file: ${specFilePath}`);
-        agentSpec = JSON.parse(readFileSync(specFilePath, 'utf8')) as AgentJobSpec;
-      } catch (err) {
-        throw SfError.create({
-          name: 'InvalidMockFile',
-          message: `SF_MOCK_DIR [${this.mockDir}] must contain a valid spec file with name ${specFileName}`,
-          cause: err,
-          actions: [
-            'Check that the file is readable',
-            'Check that the file is a valid JSON array of jobTitle and jobDescription objects',
-          ],
-        });
-      }
+    const response = await this.maybeMock.request<AgentJobSpecCreateResponse>('GET', this.buildAgentJobSpecUrl(config));
+    if (response.isSuccess && response.jobSpecs) {
+      agentSpec = response.jobSpecs;
     } else {
-      // TODO: We'll probably want to wrap this for better error handling but let's see
-      //       what it looks like first.
-      const response = await this.connection.requestGet<AgentJobSpecCreateResponse>(this.buildAgentJobSpecUrl(config), {
-        retry: { maxRetries: 3 },
+      throw SfError.create({
+        name: 'AgentJobSpecCreateError',
+        message: response.errorMessage ?? 'unknown',
       });
-      if (response.isSuccess) {
-        agentSpec = response?.jobSpecs as AgentJobSpec;
-      } else {
-        throw SfError.create({
-          name: 'AgentJobSpecCreateError',
-          message: response.errorMessage ?? 'unknown',
-        });
-      }
     }
 
     return agentSpec;

src/agentTester.ts

@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2024, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import { Connection, Lifecycle, PollingClient, StatusResult } from '@salesforce/core';
+import { Duration } from '@salesforce/kit';
+import { MaybeMock } from './maybe-mock';
+
+type Format = 'human' | 'tap' | 'junit' | 'json';
+
+type AgentTestStartResponse = {
+  id: string;
+};
+
+type AgentTestStatusResponse = {
+  status: 'NEW' | 'IN_PROGRESS' | 'COMPLETED' | 'ERROR';
+  startTime: string;
+  endTime?: string;
+  errorMessage?: string;
+};
+
+type AgentTestDetailsResponse = {
+  AiEvaluationSuiteDefinition: string;
+  tests: Array<{
+    AiEvaluationDefinition: string;
+    results: Array<{
+      test_number: number;
+      results: Array<{
+        name: string;
+        actual: string[];
+        is_pass: boolean;
+        execution_time_ms: number;
+        error?: string;
+      }>;
+    }>;
+  }>;
+};
+
+export class AgentTester {
+  private maybeMock: MaybeMock;
+  public constructor(connection: Connection) {
+    this.maybeMock = new MaybeMock(connection);
+  }
+
+  public async start(suiteId: string): Promise<{ id: string }> {
+    const url = '/einstein/ai-evaluations/runs';
+
+    return this.maybeMock.request<AgentTestStartResponse>('POST', url, {
+      aiEvaluationSuiteDefinition: suiteId,
+    });
+  }
+
+  public async status(jobId: string): Promise<AgentTestStatusResponse> {
+    const url = `/einstein/ai-evaluations/runs/${jobId}`;
+
+    return this.maybeMock.request<AgentTestStatusResponse>('GET', url);
+  }
+
+  public async poll(
+    jobId: string,
+    {
+      format = 'human',
+      timeout = Duration.minutes(5),
+    }: {
+      format?: Format;
+      timeout?: Duration;
+    } = {
+      format: 'human',
+      timeout: Duration.minutes(5),
+    }
+  ): Promise<{ response: AgentTestDetailsResponse; formatted: string }> {
+    const lifecycle = Lifecycle.getInstance();
+    const client = await PollingClient.create({
+      poll: async (): Promise<StatusResult> => {
+        const { status } = await this.status(jobId);
+        if (status === 'COMPLETED') {
+          await lifecycle.emit('AGENT_TEST_POLLING_EVENT', { jobId, status });
+          return { payload: await this.details(jobId, format), completed: true };
+        }
+
+        await lifecycle.emit('AGENT_TEST_POLLING_EVENT', { jobId, status });
+        return { completed: false };
+      },
+      frequency: Duration.seconds(1),
+      timeout,
+    });
+
+    const result = await client.subscribe<{ response: AgentTestDetailsResponse; formatted: string }>();
+    return result;
+  }
+
+  public async details(
+    jobId: string,
+    format: Format = 'human'
+  ): Promise<{ response: AgentTestDetailsResponse; formatted: string }> {
+    const url = `/einstein/ai-evaluations/runs/${jobId}/details`;
+
+    const response = await this.maybeMock.request<AgentTestDetailsResponse>('GET', url);
+    return {
+      response,
+      formatted:
+        format === 'human'
+          ? await humanFormat(response)
+          : format === 'tap'
+          ? await tapFormat(response)
+          : format === 'junit'
+          ? await junitFormat(response)
+          : await jsonFormat(response),
+    };
+  }
+}
+
+export async function humanFormat(details: AgentTestDetailsResponse): Promise<string> {
+  // TODO: these tables need to follow the same defaults that sf-plugins-core uses
+  // TODO: the api response isn't finalized so this is just a POC
+  const { makeTable } = await import('@oclif/table');
+  const tables: string[] = [];
+  for (const aiEvalDef of details.tests) {
+    for (const result of aiEvalDef.results) {
+      const table = makeTable({
+        title: `Test Results for ${aiEvalDef.AiEvaluationDefinition} (#${result.test_number})`,
+        data: result.results.map((r) => ({
+          'TEST NAME': r.name,
+          OUTCOME: r.is_pass ? 'Pass' : 'Fail',
+          MESSAGE: r.error ?? '',
+          'RUNTIME (MS)': r.execution_time_ms,
+        })),
+      });
+      tables.push(table);
+    }
+  }
+
+  return tables.join('\n');
+}
+
+export async function junitFormat(details: AgentTestDetailsResponse): Promise<string> {
+  // APEX EXAMPLE
+  // <?xml version="1.0" encoding="UTF-8"?>
+  // <testsuites>
+  //     <testsuite name="force.apex" timestamp="2024-11-13T19:19:23.000Z" hostname="https://energy-site-1368-dev-ed.scratch.my.salesforce.com" tests="11" failures="0"  errors="0"  time="2.57">
+  //         <properties>
+  //             <property name="outcome" value="Successful"/>
+  //             <property name="testsRan" value="11"/>
+  //             <property name="passing" value="11"/>
+  //             <property name="failing" value="0"/>
+  //             <property name="skipped" value="0"/>
+  //             <property name="passRate" value="100%"/>
+  //             <property name="failRate" value="0%"/>
+  //             <property name="testStartTime" value="Wed Nov 13 2024 12:19:23 PM"/>
+  //             <property name="testSetupTimeInMs" value="0"/>
+  //             <property name="testExecutionTime" value="2.57 s"/>
+  //             <property name="testTotalTime" value="2.57 s"/>
+  //             <property name="commandTime" value="0.17 s"/>
+  //             <property name="hostname" value="https://energy-site-1368-dev-ed.scratch.my.salesforce.com"/>
+  //             <property name="orgId" value="00DEi000006OlrxMAC"/>
+  //             <property name="username" value="[email protected]"/>
+  //             <property name="testRunId" value="707Ei00000dTRSa"/>
+  //             <property name="userId" value="005Ei00000FkGU9IAN"/>
+  //         </properties>
+  //         <testcase name="importSampleData" classname="TestSampleDataController" time="0.27">
+  //         </testcase>
+  //         <testcase name="blankAddress" classname="GeocodingServiceTest" time="0.01">
+  //         </testcase>
+  //         <testcase name="errorResponse" classname="GeocodingServiceTest" time="0.01">
+  //         </testcase>
+  //         <testcase name="successResponse" classname="GeocodingServiceTest" time="0.01">
+  //         </testcase>
+  //         <testcase name="createFileFailsWhenIncorrectBase64Data" classname="FileUtilitiesTest" time="0.10">
+  //         </testcase>
+  //         <testcase name="createFileFailsWhenIncorrectFilename" classname="FileUtilitiesTest" time="0.03">
+  //         </testcase>
+  //         <testcase name="createFileFailsWhenIncorrectRecordId" classname="FileUtilitiesTest" time="0.35">
+  //         </testcase>
+  //         <testcase name="createFileSucceedsWhenCorrectInput" classname="FileUtilitiesTest" time="0.22">
+  //         </testcase>
+  //         <testcase name="testGetPagedPropertyList" classname="TestPropertyController" time="1.01">
+  //         </testcase>
+  //         <testcase name="testGetPicturesNoResults" classname="TestPropertyController" time="0.06">
+  //         </testcase>
+  //         <testcase name="testGetPicturesWithResults" classname="TestPropertyController" time="0.51">
+  //         </testcase>
+  //     </testsuite>
+  // </testsuites>
+  await Promise.reject(new Error('Not implemented'));
+  return JSON.stringify(details, null, 2);
+}
+
+export async function tapFormat(details: AgentTestDetailsResponse): Promise<string> {
+  // APEX EXAMPLE
+  // 1..11
+  // ok 1 TestPropertyController.testGetPagedPropertyList
+  // ok 2 TestPropertyController.testGetPicturesNoResults
+  // ok 3 TestPropertyController.testGetPicturesWithResults
+  // ok 4 FileUtilitiesTest.createFileFailsWhenIncorrectBase64Data
+  // ok 5 FileUtilitiesTest.createFileFailsWhenIncorrectFilename
+  // ok 6 FileUtilitiesTest.createFileFailsWhenIncorrectRecordId
+  // ok 7 FileUtilitiesTest.createFileSucceedsWhenCorrectInput
+  // ok 8 TestSampleDataController.importSampleData
+  // ok 9 GeocodingServiceTest.blankAddress
+  // ok 10 GeocodingServiceTest.errorResponse
+  // ok 11 GeocodingServiceTest.successResponse
+  // # Run "sf apex get test -i 707Ei00000dUJry -o [email protected] --result-format <format>" to retrieve test results in a different format.
+  await Promise.reject(new Error('Not implemented'));
+  return JSON.stringify(details, null, 2);
+}
+
+export async function jsonFormat(details: AgentTestDetailsResponse): Promise<string> {
+  return Promise.resolve(JSON.stringify(details, null, 2));
+}

src/index.ts

@@ -14,3 +14,4 @@ export {
   SfAgent,
 } from './types';
 export { Agent } from './agent';
+export { AgentTester } from './agentTester';