From efabf4d2b991034693223a103debfd59b5074c53 Mon Sep 17 00:00:00 2001 From: Zhiming Ma Date: Wed, 1 Nov 2023 09:51:43 +0800 Subject: [PATCH 1/4] test(agent): add golden test. --- clients/tabby-agent/.prettierignore | 1 + clients/tabby-agent/package.json | 1 + clients/tabby-agent/src/TabbyAgent.ts | 5 +- clients/tabby-agent/tests/golden.test.ts | 124 ++++++++++++++++++ clients/tabby-agent/tests/golden/0-python.py | 7 + clients/tabby-agent/tests/golden/1-python.py | 16 +++ .../tabby-agent/tests/golden/2-typescript.ts | 11 ++ .../tabby-agent/tests/golden/3-typescript.ts | 6 + .../tests/server.docker-compose.yml | 17 +++ 9 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 clients/tabby-agent/.prettierignore create mode 100644 clients/tabby-agent/tests/golden.test.ts create mode 100644 clients/tabby-agent/tests/golden/0-python.py create mode 100644 clients/tabby-agent/tests/golden/1-python.py create mode 100644 clients/tabby-agent/tests/golden/2-typescript.ts create mode 100644 clients/tabby-agent/tests/golden/3-typescript.ts create mode 100644 clients/tabby-agent/tests/server.docker-compose.yml diff --git a/clients/tabby-agent/.prettierignore b/clients/tabby-agent/.prettierignore new file mode 100644 index 000000000000..fdc26c41ee79 --- /dev/null +++ b/clients/tabby-agent/.prettierignore @@ -0,0 +1 @@ +tests/golden/** \ No newline at end of file diff --git a/clients/tabby-agent/package.json b/clients/tabby-agent/package.json index 81202ab690ca..666478907c07 100644 --- a/clients/tabby-agent/package.json +++ b/clients/tabby-agent/package.json @@ -12,6 +12,7 @@ "build": "tsup", "test": "mocha", "test:watch": "env TEST_LOG_DEBUG=1 mocha --watch", + "test:golden": "mocha --grep golden ./tests/golden.test.ts", "lint": "prettier --write .", "lint:check": "prettier --check ." }, diff --git a/clients/tabby-agent/src/TabbyAgent.ts b/clients/tabby-agent/src/TabbyAgent.ts index 51361ec15ddb..6bcfb438cb1d 100644 --- a/clients/tabby-agent/src/TabbyAgent.ts +++ b/clients/tabby-agent/src/TabbyAgent.ts @@ -104,7 +104,6 @@ export class TabbyAgent extends EventEmitter implements Agent { // If auth token is provided, use it directly. this.auth = null; } - await this.setupApi(); // If server config changed, clear server related state if (!deepEqual(oldConfig.server, this.config.server)) { @@ -112,7 +111,11 @@ export class TabbyAgent extends EventEmitter implements Agent { this.completionProviderStats.resetWindowed(); this.popIssue("slowCompletionResponseTime"); this.popIssue("highCompletionTimeoutRate"); + } + await this.setupApi(); + + if (!deepEqual(oldConfig.server, this.config.server)) { // If server config changed and status remain `unauthorized`, we want to emit `authRequired` again. // but `changeStatus` will not emit `authRequired` if status is not changed, so we emit it manually here. if (oldStatus === "unauthorized" && this.status === "unauthorized") { diff --git a/clients/tabby-agent/tests/golden.test.ts b/clients/tabby-agent/tests/golden.test.ts new file mode 100644 index 000000000000..d19c8d91e0e0 --- /dev/null +++ b/clients/tabby-agent/tests/golden.test.ts @@ -0,0 +1,124 @@ +// This golden test requires local Tabby server to be running on port 8087. +// The server should use tabby linux image version 0.4.0, with model TabbyML/StarCoder-1B, +// cuda backend, and without any repository specified for RAG. +// See also `server.docker-compose.yml`. + +import { spawn } from "child_process"; +import readline from "readline"; +import path from "path"; +import * as fs from "fs-extra"; +import { expect } from "chai"; + +describe("agent golden test", () => { + const agent = spawn("node", [path.join(__dirname, "../dist/cli.js")]); + const output: any[] = []; + readline.createInterface({ input: agent.stdout }).on("line", (line) => { + output.push(JSON.parse(line)); + }); + + const waitForResponse = (requestId, timeout = 1000) => { + return new Promise((resolve, reject) => { + const start = Date.now(); + const interval = setInterval(() => { + if (output.find((item) => item[0] === requestId)) { + clearInterval(interval); + resolve(); + } else if (Date.now() - start > timeout) { + clearInterval(interval); + reject(new Error("Timeout")); + } + }, 10); + }); + }; + + const createGoldenTest = async (goldenFilepath) => { + const content = await fs.readFile(goldenFilepath, "utf8"); + const language = path.basename(goldenFilepath, path.extname(goldenFilepath)).replace(/^\d+-/g, ""); + const replaceStart = content.indexOf("⏩"); + const insertStart = content.indexOf("⏭"); + const insertEnd = content.indexOf("⏮"); + const replaceEnd = content.indexOf("⏪"); + const prefix = content.slice(0, replaceStart); + const replacePrefix = content.slice(replaceStart + 1, insertStart); + const suggestion = content.slice(insertStart + 1, insertEnd); + const replaceSuffix = content.slice(insertEnd + 1, replaceEnd); + const suffix = content.slice(replaceEnd + 1); + const request = { + filepath: goldenFilepath, + language, + text: prefix + replacePrefix + replaceSuffix + suffix, + position: prefix.length + replacePrefix.length, + manually: true, + }; + const expected = { + choices: [ + { + index: 0, + text: replacePrefix + suggestion, + replaceRange: { + start: prefix.length, + end: prefix.length + replacePrefix.length + replaceSuffix.length, + }, + }, + ], + }; + return { request, expected }; + }; + + it("initialize", async () => { + const requestId = 1; + const config = { + server: { + endpoint: "http://127.0.0.1:8087", + token: "", + requestHeaders: {}, + requestTimeout: 30000, + }, + completion: { + prompt: { experimentalStripAutoClosingCharacters: false, maxPrefixLines: 20, maxSuffixLines: 20 }, + debounce: { mode: "adaptive", interval: 250 }, + timeout: { auto: 4000, manually: 4000 }, + }, + postprocess: { limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false } }, + logs: { level: "debug" }, + anonymousUsageTracking: { disable: true }, + }; + const initRequest = [ + requestId, + { + func: "initialize", + args: [{ config }], + }, + ]; + + agent.stdin.write(JSON.stringify(initRequest) + "\n"); + await waitForResponse(requestId); + expect(output.shift()).to.deep.equal([0, { event: "statusChanged", status: "ready" }]); + expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config }]); + expect(output.shift()).to.deep.equal([requestId, true]); + }); + + const goldenFiles = fs.readdirSync(path.join(__dirname, "golden")).map((file) => { + return { + name: file, + path: path.join(__dirname, "golden", file), + }; + }); + const baseRequestId = 2; + goldenFiles.forEach((goldenFile, index) => { + it(goldenFile.name, async () => { + const test = await createGoldenTest(goldenFile.path); + const requestId = baseRequestId + index; + const request = [requestId, { func: "provideCompletions", args: [test.request] }]; + agent.stdin.write(JSON.stringify(request) + "\n"); + await waitForResponse(requestId); + const response = output.shift(); + expect(response[0]).to.equal(requestId); + expect(response[1].choices).to.deep.equal(test.expected.choices); + }); + }); + + after(() => { + agent.kill(); + }); +}); diff --git a/clients/tabby-agent/tests/golden/0-python.py b/clients/tabby-agent/tests/golden/0-python.py new file mode 100644 index 000000000000..439ed407a50d --- /dev/null +++ b/clients/tabby-agent/tests/golden/0-python.py @@ -0,0 +1,7 @@ +def fib(n): + ⏩⏭if n == 0: + return 0 + elif n == 1: + return 1 + else:⏮⏪ + return fib(n - 1) + fib(n - 2) \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/1-python.py b/clients/tabby-agent/tests/golden/1-python.py new file mode 100644 index 000000000000..3e1a75a4777d --- /dev/null +++ b/clients/tabby-agent/tests/golden/1-python.py @@ -0,0 +1,16 @@ +import datetime + +def parse_expenses(expenses_string): + """Parse the list of expenses and return the list of triples (date, value, currency). + Ignore lines starting with #. + Parse the date using datetime. + Example expenses_string: + 2016-01-02 -34.01 USD + 2016-01-03 2.59 DKK + 2016-01-03 -2.72 EUR + """ + for line in expenses_string.split('\\n'): + ⏩⏭if line.startswith('#'): + continue + date, value, currency = line.split() + yield datetime.datetime.strptime(date, '%Y-%m-%d'), float(value), currency⏮⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/2-typescript.ts b/clients/tabby-agent/tests/golden/2-typescript.ts new file mode 100644 index 000000000000..721c8ddc3b09 --- /dev/null +++ b/clients/tabby-agent/tests/golden/2-typescript.ts @@ -0,0 +1,11 @@ +export class Foo { + private _foo: number; + + constructor() { + this._foo = 1; + } + + update(value): Foo { + this._foo = max(⏩⏭this._foo, value⏮⏪) + } +} diff --git a/clients/tabby-agent/tests/golden/3-typescript.ts b/clients/tabby-agent/tests/golden/3-typescript.ts new file mode 100644 index 000000000000..2a71f6c7a6b4 --- /dev/null +++ b/clients/tabby-agent/tests/golden/3-typescript.ts @@ -0,0 +1,6 @@ +function fib(⏩⏭n) { + if (n < 2) { + return n; + } + return fib(n - 1) + fib(n - 2); +}⏮)⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/server.docker-compose.yml b/clients/tabby-agent/tests/server.docker-compose.yml new file mode 100644 index 000000000000..45885fc35523 --- /dev/null +++ b/clients/tabby-agent/tests/server.docker-compose.yml @@ -0,0 +1,17 @@ +version: "3.5" +services: + serve: + restart: always + image: tabbyml/tabby:0.4.0 + command: serve --model "/data/models/TabbyML/SantaCoder-1B" --device cuda + ports: + - "8087:8080" + volumes: + - "~/.tabby:/data" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] From 3ea14929a8cb6b33fce0f39a52252673b1d4c5d1 Mon Sep 17 00:00:00 2001 From: Zhiming Ma Date: Sat, 4 Nov 2023 08:13:42 +0800 Subject: [PATCH 2/4] test(agent): update golden test. --- clients/tabby-agent/tests/golden.test.ts | 29 ++++++++++--------- clients/tabby-agent/tests/golden/0-python.py | 9 +++--- clients/tabby-agent/tests/golden/1-python.py | 5 ++-- .../tabby-agent/tests/golden/3-typescript.ts | 7 +---- .../tabby-agent/tests/golden/4-typescript.ts | 9 ++++++ 5 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 clients/tabby-agent/tests/golden/4-typescript.ts diff --git a/clients/tabby-agent/tests/golden.test.ts b/clients/tabby-agent/tests/golden.test.ts index d19c8d91e0e0..a009a49ee4d7 100644 --- a/clients/tabby-agent/tests/golden.test.ts +++ b/clients/tabby-agent/tests/golden.test.ts @@ -1,7 +1,6 @@ // This golden test requires local Tabby server to be running on port 8087. // The server should use tabby linux image version 0.4.0, with model TabbyML/StarCoder-1B, // cuda backend, and without any repository specified for RAG. -// See also `server.docker-compose.yml`. import { spawn } from "child_process"; import readline from "readline"; @@ -50,18 +49,22 @@ describe("agent golden test", () => { position: prefix.length + replacePrefix.length, manually: true, }; - const expected = { - choices: [ - { - index: 0, - text: replacePrefix + suggestion, - replaceRange: { - start: prefix.length, - end: prefix.length + replacePrefix.length + replaceSuffix.length, - }, - }, - ], - }; + const text = replacePrefix + suggestion; + const expected = + text.length == 0 + ? { choices: [] } + : { + choices: [ + { + index: 0, + text, + replaceRange: { + start: prefix.length, + end: prefix.length + replacePrefix.length + replaceSuffix.length, + }, + }, + ], + }; return { request, expected }; }; diff --git a/clients/tabby-agent/tests/golden/0-python.py b/clients/tabby-agent/tests/golden/0-python.py index 439ed407a50d..2bf4bb520c6d 100644 --- a/clients/tabby-agent/tests/golden/0-python.py +++ b/clients/tabby-agent/tests/golden/0-python.py @@ -1,7 +1,6 @@ def fib(n): - ⏩⏭if n == 0: - return 0 - elif n == 1: - return 1 - else:⏮⏪ + ⏩⏭ if n == 0: + return 0 + if n == 1: + return 1⏮⏪ return fib(n - 1) + fib(n - 2) \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/1-python.py b/clients/tabby-agent/tests/golden/1-python.py index 3e1a75a4777d..f98735bd168a 100644 --- a/clients/tabby-agent/tests/golden/1-python.py +++ b/clients/tabby-agent/tests/golden/1-python.py @@ -12,5 +12,6 @@ def parse_expenses(expenses_string): for line in expenses_string.split('\\n'): ⏩⏭if line.startswith('#'): continue - date, value, currency = line.split() - yield datetime.datetime.strptime(date, '%Y-%m-%d'), float(value), currency⏮⏪ \ No newline at end of file + date, value, currency = line.split(' - ') + date = datetime.datetime.strptime(date, '%Y-%m-%d') + yield date, float(value), currency⏮⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/3-typescript.ts b/clients/tabby-agent/tests/golden/3-typescript.ts index 2a71f6c7a6b4..7bc37f2c0a34 100644 --- a/clients/tabby-agent/tests/golden/3-typescript.ts +++ b/clients/tabby-agent/tests/golden/3-typescript.ts @@ -1,6 +1 @@ -function fib(⏩⏭n) { - if (n < 2) { - return n; - } - return fib(n - 1) + fib(n - 2); -}⏮)⏪ \ No newline at end of file +function fibonacci(⏩⏭⏮⏪) \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/4-typescript.ts b/clients/tabby-agent/tests/golden/4-typescript.ts new file mode 100644 index 000000000000..dd4ad0447908 --- /dev/null +++ b/clients/tabby-agent/tests/golden/4-typescript.ts @@ -0,0 +1,9 @@ +function fibonacci(⏩⏭n) { + if (n === 0) { + return 0; + } + if (n === 1) { + return 1; + } + return fibonacci(n - 1) + fibonacci(n - 2); +}⏮⏪ \ No newline at end of file From df7f7f8129be13bbc43abf14b4e14fcdf0ed29a5 Mon Sep 17 00:00:00 2001 From: Zhiming Ma Date: Sat, 4 Nov 2023 10:33:57 +0800 Subject: [PATCH 3/4] test(agent): remove docker-compose. --- .../tabby-agent/tests/server.docker-compose.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 clients/tabby-agent/tests/server.docker-compose.yml diff --git a/clients/tabby-agent/tests/server.docker-compose.yml b/clients/tabby-agent/tests/server.docker-compose.yml deleted file mode 100644 index 45885fc35523..000000000000 --- a/clients/tabby-agent/tests/server.docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.5" -services: - serve: - restart: always - image: tabbyml/tabby:0.4.0 - command: serve --model "/data/models/TabbyML/SantaCoder-1B" --device cuda - ports: - - "8087:8080" - volumes: - - "~/.tabby:/data" - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu] From a69e49fffe2ae62d141a5336fba0409a04da9fa8 Mon Sep 17 00:00:00 2001 From: Zhiming Ma Date: Sun, 5 Nov 2023 09:47:51 +0800 Subject: [PATCH 4/4] test(agent): update golden test. --- clients/tabby-agent/tests/golden.test.ts | 7 ++++++- clients/tabby-agent/tests/golden/5-go.go | 9 +++++++++ clients/tabby-agent/tests/golden/6-rust.rs | 9 +++++++++ clients/tabby-agent/tests/golden/7-ruby.rb | 4 ++++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 clients/tabby-agent/tests/golden/5-go.go create mode 100644 clients/tabby-agent/tests/golden/6-rust.rs create mode 100644 clients/tabby-agent/tests/golden/7-ruby.rb diff --git a/clients/tabby-agent/tests/golden.test.ts b/clients/tabby-agent/tests/golden.test.ts index a009a49ee4d7..6e7b63ffd32b 100644 --- a/clients/tabby-agent/tests/golden.test.ts +++ b/clients/tabby-agent/tests/golden.test.ts @@ -82,7 +82,12 @@ describe("agent golden test", () => { debounce: { mode: "adaptive", interval: 250 }, timeout: { auto: 4000, manually: 4000 }, }, - postprocess: { limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false } }, + postprocess: { + limitScope: { + experimentalSyntax: false, + indentation: { experimentalKeepBlockScopeWhenCompletingLine: false }, + }, + }, logs: { level: "debug" }, anonymousUsageTracking: { disable: true }, }; diff --git a/clients/tabby-agent/tests/golden/5-go.go b/clients/tabby-agent/tests/golden/5-go.go new file mode 100644 index 000000000000..f9abe15f5acb --- /dev/null +++ b/clients/tabby-agent/tests/golden/5-go.go @@ -0,0 +1,9 @@ +func fibonacci(⏩⏭n int) int { + if n == 0 { + return 0 + } + if n == 1 { + return 1 + } + return fibonacci(n-1) + fibonacci(n-2) +}⏮⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/6-rust.rs b/clients/tabby-agent/tests/golden/6-rust.rs new file mode 100644 index 000000000000..67fcf41a1997 --- /dev/null +++ b/clients/tabby-agent/tests/golden/6-rust.rs @@ -0,0 +1,9 @@ +fn fibonacci(⏩⏭n: number): number { + if (n === 0) { + return 0; + } + if (n === 1) { + return 1; + } + return fibonacci(n - 1) + fibonacci(n - 2); +}⏮⏪ \ No newline at end of file diff --git a/clients/tabby-agent/tests/golden/7-ruby.rb b/clients/tabby-agent/tests/golden/7-ruby.rb new file mode 100644 index 000000000000..f84f50ea70dc --- /dev/null +++ b/clients/tabby-agent/tests/golden/7-ruby.rb @@ -0,0 +1,4 @@ +def fibonacci(n) + return n if n <= 1 + ⏩⏭fibonacci(n - 1) + fibonacci(n - 2)⏮⏪ +end \ No newline at end of file