From 43a2f27d4a4e637d767c0a0ae44f3b5da867c977 Mon Sep 17 00:00:00 2001 From: Zhiming Ma Date: Wed, 1 Nov 2023 09:51:43 +0800 Subject: [PATCH] test(agent): add golden test. --- clients/tabby-agent/.prettierignore | 1 + clients/tabby-agent/package.json | 1 + clients/tabby-agent/src/TabbyAgent.ts | 5 +- clients/tabby-agent/tests/golden.test.ts | 124 ++++++++++++++++++ clients/tabby-agent/tests/golden/0-python.py | 7 + clients/tabby-agent/tests/golden/1-python.py | 16 +++ .../tabby-agent/tests/golden/2-typescript.ts | 11 ++ .../tabby-agent/tests/golden/3-typescript.ts | 6 + .../tests/server.docker-compose.yml | 17 +++ 9 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 clients/tabby-agent/.prettierignore create mode 100644 clients/tabby-agent/tests/golden.test.ts create mode 100644 clients/tabby-agent/tests/golden/0-python.py create mode 100644 clients/tabby-agent/tests/golden/1-python.py create mode 100644 clients/tabby-agent/tests/golden/2-typescript.ts create mode 100644 clients/tabby-agent/tests/golden/3-typescript.ts create mode 100644 clients/tabby-agent/tests/server.docker-compose.yml diff --git a/clients/tabby-agent/.prettierignore b/clients/tabby-agent/.prettierignore new file mode 100644 index 000000000000..fdc26c41ee79 --- /dev/null +++ b/clients/tabby-agent/.prettierignore @@ -0,0 +1 @@ +tests/golden/** \ No newline at end of file diff --git a/clients/tabby-agent/package.json b/clients/tabby-agent/package.json index 81202ab690ca..666478907c07 100644 --- a/clients/tabby-agent/package.json +++ b/clients/tabby-agent/package.json @@ -12,6 +12,7 @@ "build": "tsup", "test": "mocha", "test:watch": "env TEST_LOG_DEBUG=1 mocha --watch", + "test:golden": "mocha --grep golden ./tests/golden.test.ts", "lint": "prettier --write .", "lint:check": "prettier --check ." }, diff --git a/clients/tabby-agent/src/TabbyAgent.ts b/clients/tabby-agent/src/TabbyAgent.ts index 51361ec15ddb..6bcfb438cb1d 100644 --- a/clients/tabby-agent/src/TabbyAgent.ts +++ b/clients/tabby-agent/src/TabbyAgent.ts @@ -104,7 +104,6 @@ export class TabbyAgent extends EventEmitter implements Agent { // If auth token is provided, use it directly. this.auth = null; } - await this.setupApi(); // If server config changed, clear server related state if (!deepEqual(oldConfig.server, this.config.server)) { @@ -112,7 +111,11 @@ export class TabbyAgent extends EventEmitter implements Agent { this.completionProviderStats.resetWindowed(); this.popIssue("slowCompletionResponseTime"); this.popIssue("highCompletionTimeoutRate"); + } + await this.setupApi(); + + if (!deepEqual(oldConfig.server, this.config.server)) { // If server config changed and status remain `unauthorized`, we want to emit `authRequired` again. // but `changeStatus` will not emit `authRequired` if status is not changed, so we emit it manually here. if (oldStatus === "unauthorized" && this.status === "unauthorized") { diff --git a/clients/tabby-agent/tests/golden.test.ts b/clients/tabby-agent/tests/golden.test.ts new file mode 100644 index 000000000000..d19c8d91e0e0 --- /dev/null +++ b/clients/tabby-agent/tests/golden.test.ts @@ -0,0 +1,124 @@ +// This golden test requires local Tabby server to be running on port 8087. +// The server should use tabby linux image version 0.4.0, with model TabbyML/StarCoder-1B, +// cuda backend, and without any repository specified for RAG. +// See also `server.docker-compose.yml`. + +import { spawn } from "child_process"; +import readline from "readline"; +import path from "path"; +import * as fs from "fs-extra"; +import { expect } from "chai"; + +describe("agent golden test", () => { + const agent = spawn("node", [path.join(__dirname, "../dist/cli.js")]); + const output: any[] = []; + readline.createInterface({ input: agent.stdout }).on("line", (line) => { + output.push(JSON.parse(line)); + }); + + const waitForResponse = (requestId, timeout = 1000) => { + return new Promise((resolve, reject) => { + const start = Date.now(); + const interval = setInterval(() => { + if (output.find((item) => item[0] === requestId)) { + clearInterval(interval); + resolve(); + } else if (Date.now() - start > timeout) { + clearInterval(interval); + reject(new Error("Timeout")); + } + }, 10); + }); + }; + + const createGoldenTest = async (goldenFilepath) => { + const content = await fs.readFile(goldenFilepath, "utf8"); + const language = path.basename(goldenFilepath, path.extname(goldenFilepath)).replace(/^\d+-/g, ""); + const replaceStart = content.indexOf("⏩"); + const insertStart = content.indexOf("⏭"); + const insertEnd = content.indexOf("⏮"); + const replaceEnd = content.indexOf("⏪"); + const prefix = content.slice(0, replaceStart); + const replacePrefix = content.slice(replaceStart + 1, insertStart); + const suggestion = content.slice(insertStart + 1, insertEnd); + const replaceSuffix = content.slice(insertEnd + 1, replaceEnd); + const suffix = content.slice(replaceEnd + 1); + const request = { + filepath: goldenFilepath, + language, + text: prefix + replacePrefix + replaceSuffix + suffix, + position: prefix.length + replacePrefix.length, + manually: true, + }; + const expected = { + choices: [ + { + index: 0, + text: replacePrefix + suggestion, + replaceRange: { + start: prefix.length, + end: prefix.length + replacePrefix.length + replaceSuffix.length, + }, + }, + ], + }; + return { request, expected }; + }; + + it("initialize", async () => { + const requestId = 1; + const config = { + server: { + endpoint: "http://127.0.0.1:8087", + token: "", + requestHeaders: {}, + requestTimeout: 30000, + }, + completion: { + prompt: { experimentalStripAutoClosingCharacters: false, maxPrefixLines: 20, maxSuffixLines: 20 }, + debounce: { mode: "adaptive", interval: 250 }, + timeout: { auto: 4000, manually: 4000 }, + }, + postprocess: { limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false } }, + logs: { level: "debug" }, + anonymousUsageTracking: { disable: true }, + }; + const initRequest = [ + requestId, + { + func: "initialize", + args: [{ config }], + }, + ]; + + agent.stdin.write(JSON.stringify(initRequest) + "\n"); + await waitForResponse(requestId); + expect(output.shift()).to.deep.equal([0, { event: "statusChanged", status: "ready" }]); + expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config }]); + expect(output.shift()).to.deep.equal([requestId, true]); + }); + + const goldenFiles = fs.readdirSync(path.join(__dirname, "golden")).map((file) => { + return { + name: file, + path: path.join(__dirname, "golden", file), + }; + }); + const baseRequestId = 2; + goldenFiles.forEach((goldenFile, index) => { + it(goldenFile.name, async () => { + const test = await createGoldenTest(goldenFile.path); + const requestId = baseRequestId + index; + const request = [requestId, { func: "provideCompletions", args: [test.request] }]; + agent.stdin.write(JSON.stringify(request) + "\n"); + await waitForResponse(requestId); + const response = output.shift(); + expect(response[0]).to.equal(requestId); + expect(response[1].choices).to.deep.equal(test.expected.choices); + }); + }); + + after(() => { + agent.kill(); + }); +}); diff --git a/clients/tabby-agent/tests/golden/0-python.py b/clients/tabby-agent/tests/golden/0-python.py new file mode 100644 index 000000000000..439ed407a50d --- /dev/null +++ b/clients/tabby-agent/tests/golden/0-python.py @@ -0,0 +1,7 @@ +def fib(n): + ⏩⏭if n == 0: + return 0 + elif n == 1: + return 1 + else:⏮⏪ + return fib(n - 1) + fib(n - 2) \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/1-python.py b/clients/tabby-agent/tests/golden/1-python.py new file mode 100644 index 000000000000..3e1a75a4777d --- /dev/null +++ b/clients/tabby-agent/tests/golden/1-python.py @@ -0,0 +1,16 @@ +import datetime + +def parse_expenses(expenses_string): + """Parse the list of expenses and return the list of triples (date, value, currency). + Ignore lines starting with #. + Parse the date using datetime. + Example expenses_string: + 2016-01-02 -34.01 USD + 2016-01-03 2.59 DKK + 2016-01-03 -2.72 EUR + """ + for line in expenses_string.split('\\n'): + ⏩⏭if line.startswith('#'): + continue + date, value, currency = line.split() + yield datetime.datetime.strptime(date, '%Y-%m-%d'), float(value), currency⏮⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/2-typescript.ts b/clients/tabby-agent/tests/golden/2-typescript.ts new file mode 100644 index 000000000000..721c8ddc3b09 --- /dev/null +++ b/clients/tabby-agent/tests/golden/2-typescript.ts @@ -0,0 +1,11 @@ +export class Foo { + private _foo: number; + + constructor() { + this._foo = 1; + } + + update(value): Foo { + this._foo = max(⏩⏭this._foo, value⏮⏪) + } +} diff --git a/clients/tabby-agent/tests/golden/3-typescript.ts b/clients/tabby-agent/tests/golden/3-typescript.ts new file mode 100644 index 000000000000..2a71f6c7a6b4 --- /dev/null +++ b/clients/tabby-agent/tests/golden/3-typescript.ts @@ -0,0 +1,6 @@ +function fib(⏩⏭n) { + if (n < 2) { + return n; + } + return fib(n - 1) + fib(n - 2); +}⏮)⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/server.docker-compose.yml b/clients/tabby-agent/tests/server.docker-compose.yml new file mode 100644 index 000000000000..45885fc35523 --- /dev/null +++ b/clients/tabby-agent/tests/server.docker-compose.yml @@ -0,0 +1,17 @@ +version: "3.5" +services: + serve: + restart: always + image: tabbyml/tabby:0.4.0 + command: serve --model "/data/models/TabbyML/SantaCoder-1B" --device cuda + ports: + - "8087:8080" + volumes: + - "~/.tabby:/data" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu]