test(agent): add golden test.

clients/tabby-agent/.prettierignore

@@ -0,0 +1 @@
+tests/golden/**

clients/tabby-agent/package.json

@@ -12,6 +12,7 @@
     "build": "tsup",
     "test": "mocha",
     "test:watch": "env TEST_LOG_DEBUG=1 mocha --watch",
+    "test:golden": "mocha --grep golden ./tests/golden.test.ts",
     "lint": "prettier --write .",
     "lint:check": "prettier --check ."
   },

clients/tabby-agent/src/TabbyAgent.ts

@@ -104,15 +104,18 @@ export class TabbyAgent extends EventEmitter implements Agent {
       // If auth token is provided, use it directly.
       this.auth = null;
     }
-    await this.setupApi();
 
     // If server config changed, clear server related state
     if (!deepEqual(oldConfig.server, this.config.server)) {
       this.serverHealthState = null;
       this.completionProviderStats.resetWindowed();
       this.popIssue("slowCompletionResponseTime");
       this.popIssue("highCompletionTimeoutRate");
+    }
 
+    await this.setupApi();
+
+    if (!deepEqual(oldConfig.server, this.config.server)) {
       // If server config changed and status remain `unauthorized`, we want to emit `authRequired` again.
       // but `changeStatus` will not emit `authRequired` if status is not changed, so we emit it manually here.
       if (oldStatus === "unauthorized" && this.status === "unauthorized") {

clients/tabby-agent/tests/golden.test.ts

@@ -0,0 +1,124 @@
+// This golden test requires local Tabby server to be running on port 8087.
+// The server should use tabby linux image version 0.4.0, with model TabbyML/StarCoder-1B,
+// cuda backend, and without any repository specified for RAG.
+// See also `server.docker-compose.yml`.
+
+import { spawn } from "child_process";
+import readline from "readline";
+import path from "path";
+import * as fs from "fs-extra";
+import { expect } from "chai";
+
+describe("agent golden test", () => {
+  const agent = spawn("node", [path.join(__dirname, "../dist/cli.js")]);
+  const output: any[] = [];
+  readline.createInterface({ input: agent.stdout }).on("line", (line) => {
+    output.push(JSON.parse(line));
+  });
+
+  const waitForResponse = (requestId, timeout = 1000) => {
+    return new Promise<void>((resolve, reject) => {
+      const start = Date.now();
+      const interval = setInterval(() => {
+        if (output.find((item) => item[0] === requestId)) {
+          clearInterval(interval);
+          resolve();
+        } else if (Date.now() - start > timeout) {
+          clearInterval(interval);
+          reject(new Error("Timeout"));
+        }
+      }, 10);
+    });
+  };
+
+  const createGoldenTest = async (goldenFilepath) => {
+    const content = await fs.readFile(goldenFilepath, "utf8");
+    const language = path.basename(goldenFilepath, path.extname(goldenFilepath)).replace(/^\d+-/g, "");
+    const replaceStart = content.indexOf("⏩");
+    const insertStart = content.indexOf("⏭");
+    const insertEnd = content.indexOf("⏮");
+    const replaceEnd = content.indexOf("⏪");
+    const prefix = content.slice(0, replaceStart);
+    const replacePrefix = content.slice(replaceStart + 1, insertStart);
+    const suggestion = content.slice(insertStart + 1, insertEnd);
+    const replaceSuffix = content.slice(insertEnd + 1, replaceEnd);
+    const suffix = content.slice(replaceEnd + 1);
+    const request = {
+      filepath: goldenFilepath,
+      language,
+      text: prefix + replacePrefix + replaceSuffix + suffix,
+      position: prefix.length + replacePrefix.length,
+      manually: true,
+    };
+    const expected = {
+      choices: [
+        {
+          index: 0,
+          text: replacePrefix + suggestion,
+          replaceRange: {
+            start: prefix.length,
+            end: prefix.length + replacePrefix.length + replaceSuffix.length,
+          },
+        },
+      ],
+    };
+    return { request, expected };
+  };
+
+  it("initialize", async () => {
+    const requestId = 1;
+    const config = {
+      server: {
+        endpoint: "http://127.0.0.1:8087",
+        token: "",
+        requestHeaders: {},
+        requestTimeout: 30000,
+      },
+      completion: {
+        prompt: { experimentalStripAutoClosingCharacters: false, maxPrefixLines: 20, maxSuffixLines: 20 },
+        debounce: { mode: "adaptive", interval: 250 },
+        timeout: { auto: 4000, manually: 4000 },
+      },
+      postprocess: { limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false } },
+      logs: { level: "debug" },
+      anonymousUsageTracking: { disable: true },
+    };
+    const initRequest = [
+      requestId,
+      {
+        func: "initialize",
+        args: [{ config }],
+      },
+    ];
+
+    agent.stdin.write(JSON.stringify(initRequest) + "\n");
+    await waitForResponse(requestId);
+    expect(output.shift()).to.deep.equal([0, { event: "statusChanged", status: "ready" }]);
+    expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config }]);
+    expect(output.shift()).to.deep.equal([requestId, true]);
+  });
+
+  const goldenFiles = fs.readdirSync(path.join(__dirname, "golden")).map((file) => {
+    return {
+      name: file,
+      path: path.join(__dirname, "golden", file),
+    };
+  });
+  const baseRequestId = 2;
+  goldenFiles.forEach((goldenFile, index) => {
+    it(goldenFile.name, async () => {
+      const test = await createGoldenTest(goldenFile.path);
+      const requestId = baseRequestId + index;
+      const request = [requestId, { func: "provideCompletions", args: [test.request] }];
+      agent.stdin.write(JSON.stringify(request) + "\n");
+      await waitForResponse(requestId);
+      const response = output.shift();
+      expect(response[0]).to.equal(requestId);
+      expect(response[1].choices).to.deep.equal(test.expected.choices);
+    });
+  });
+
+  after(() => {
+    agent.kill();
+  });
+});

clients/tabby-agent/tests/golden/0-python.py

@@ -0,0 +1,7 @@
+def fib(n):
+    ⏩⏭if n == 0:
+        return 0
+    elif n == 1:
+        return 1
+    else:⏮⏪
+        return fib(n - 1) + fib(n - 2)

clients/tabby-agent/tests/golden/1-python.py

@@ -0,0 +1,16 @@
+import datetime
+
+def parse_expenses(expenses_string):
+    """Parse the list of expenses and return the list of triples (date, value, currency).
+    Ignore lines starting with #.
+    Parse the date using datetime.
+    Example expenses_string:
+        2016-01-02 -34.01 USD
+        2016-01-03 2.59 DKK
+        2016-01-03 -2.72 EUR
+    """
+    for line in expenses_string.split('\\n'):
+        ⏩⏭if line.startswith('#'):
+            continue
+        date, value, currency = line.split()
+        yield datetime.datetime.strptime(date, '%Y-%m-%d'), float(value), currency⏮⏪

clients/tabby-agent/tests/golden/2-typescript.ts

@@ -0,0 +1,11 @@
+export class Foo {
+  private _foo: number;
+
+  constructor() {
+    this._foo = 1;
+  }
+
+  update(value): Foo {
+    this._foo = max(⏩⏭this._foo, value⏮⏪)
+  }
+}

clients/tabby-agent/tests/golden/3-typescript.ts

@@ -0,0 +1,6 @@
+function fib(⏩⏭n) {
+  if (n < 2) {
+    return n;
+  }
+  return fib(n - 1) + fib(n - 2);
+}⏮)⏪

clients/tabby-agent/tests/server.docker-compose.yml

@@ -0,0 +1,17 @@
+version: "3.5"
+services:
+  serve:
+    restart: always
+    image: tabbyml/tabby:0.4.0
+    command: serve --model "/data/models/TabbyML/SantaCoder-1B" --device cuda
+    ports:
+      - "8087:8080"
+    volumes:
+      - "~/.tabby:/data"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]