Improve and fix chip-tests (#431)

project-chip · Oct 24, 2023 · f15e295 · f15e295
1 parent 91bf399
commit f15e295
Show file tree

Hide file tree

Showing 15 changed files with 48 additions and 43 deletions.
diff --git a/chip-testing/src/DeviceTestInstance.ts b/chip-testing/src/DeviceTestInstance.ts
@@ -106,12 +106,12 @@ export abstract class DeviceTestInstance {
     }
 
     /** Handle a user prompt from the chip-tool. The response is the answer to give. */
-    async handleUserprompt(userPrompt: string, testDescription: string): Promise<string> {
+    async handleUserprompt(testDescription: string, userPrompt: string): Promise<string> {
         if (testDescription.includes("TH reads")) {
             return "y\n"; // We acknowledge the TH reads as checked
         }
         throw new Error(
-            `Test instance ${this.testName} do not know how to handle Userprompt ${testDescription} with prompt: ${userPrompt}`,
+            `Test instance ${this.testName} do not know how to handle Userprompt ${userPrompt} with description: ${testDescription}`,
         );
     }
 }
diff --git a/chip-testing/src/suites-long/Test_TC_CADMIN_1_4.ts b/chip-testing/src/suites-long/Test_TC_CADMIN_1_4.ts
@@ -15,14 +15,14 @@ export class Test_TC_CADMIN_1_4 extends MinimalOnOffDeviceTestInstance {
         super(overrideTestName ?? "Test_TC_CADMIN_1_4", "GeneralTestPicsFile.txt", storage);
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Reset Devices to factory defaults")) {
             return "y\n"; // Already done automatically
         } else if (testDescription.includes("opens a commissioning window")) {
             return "y\n";
         } else if (testDescription.includes("Verify")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_BINFO_1_1.ts b/chip-testing/src/suites/Test_TC_BINFO_1_1.ts
@@ -56,10 +56,10 @@ export class Test_TC_BINFO_1_1 extends DeviceTestInstance {
         return this.commissioningServer;
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("TH reads")) {
             return "y\n"; // We acknowledge the TH reads as checked
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_BINFO_2_1.ts b/chip-testing/src/suites/Test_TC_BINFO_2_1.ts
@@ -17,7 +17,7 @@ export class Test_TC_BINFO_2_1 extends Test_TC_BINFO_1_1 {
         super(storage, "Test_TC_BINFO_2_1");
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("TH Saves the CountryCode Information")) {
             return "y\n";
         }
@@ -32,6 +32,6 @@ export class Test_TC_BINFO_2_1 extends Test_TC_BINFO_1_1 {
         if (testDescription.includes("Verify ")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_BINFO_2_2.ts b/chip-testing/src/suites/Test_TC_BINFO_2_2.ts
@@ -16,7 +16,7 @@ export class Test_TC_BINFO_2_2 extends Test_TC_BINFO_1_1 {
         super(storage, "Test_TC_BINFO_2_2");
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Reboot target device(DUT)")) {
             return "y\n";
         }
@@ -28,6 +28,6 @@ export class Test_TC_BINFO_2_2 extends Test_TC_BINFO_1_1 {
         if (testDescription.includes("Verify that ShutDown event has priority set as CRITICAL")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_BRBINFO_1_1.ts b/chip-testing/src/suites/Test_TC_BRBINFO_1_1.ts
@@ -70,13 +70,13 @@ export class Test_TC_BRBINFO_1_1 extends DeviceTestInstance {
         return this.commissioningServer;
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Read EventList attribute from the DUT")) {
             return "y\n";
         }
         if (testDescription.includes("TH1 reads")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_BRBINFO_2_1.ts b/chip-testing/src/suites/Test_TC_BRBINFO_2_1.ts
@@ -16,7 +16,7 @@ export class Test_TC_BRBINFO_2_1 extends Test_TC_BRBINFO_1_1 {
         super(storage, "Test_TC_BRBINFO_2_1");
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Verify that VendorID matches the value assigned to this manufacturer")) {
             return "y\n";
         }
@@ -32,6 +32,6 @@ export class Test_TC_BRBINFO_2_1 extends Test_TC_BRBINFO_1_1 {
         if (testDescription.includes("Verify that it does not include the name of the vendor")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_CADMIN_1_16.ts b/chip-testing/src/suites/Test_TC_CADMIN_1_16.ts
@@ -15,10 +15,10 @@ export class Test_TC_CADMIN_1_16 extends MinimalOnOffDeviceTestInstance {
         super(overrideTestName ?? "Test_TC_CADMIN_1_16", "GeneralTestPicsFile.txt", storage);
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Step 8b:")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_DESC_1_1.ts b/chip-testing/src/suites/Test_TC_DESC_1_1.ts
@@ -15,10 +15,10 @@ export class Test_TC_DESC_1_1 extends MinimalOnOffDeviceTestInstance {
         super("Test_TC_DESC_1_1", "GeneralTestPicsFile.txt", storage);
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Read the global attribute")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_FLABEL_1_1.ts b/chip-testing/src/suites/Test_TC_FLABEL_1_1.ts
@@ -49,10 +49,10 @@ export class Test_TC_FLABEL_1_1 extends DeviceTestInstance {
         return this.commissioningServer;
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("TH reads")) {
             return "y\n"; // We acknowledge the TH reads as checked
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_FLABEL_2_1.ts b/chip-testing/src/suites/Test_TC_FLABEL_2_1.ts
@@ -17,7 +17,7 @@ export class Test_TC_FLABEL_2_1 extends Test_TC_FLABEL_1_1 {
         super(storage, "Test_TC_FLABEL_2_1");
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("TH reads LabelList from the DUT")) {
             const fixedLabelCluster = this.onOffDevice.getClusterServer(FixedLabel.Cluster);
             const labelList = fixedLabelCluster?.getLabelListAttribute();
@@ -31,6 +31,6 @@ export class Test_TC_FLABEL_2_1 extends Test_TC_FLABEL_1_1 {
             }
             return "n\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_OO_1_1.ts b/chip-testing/src/suites/Test_TC_OO_1_1.ts
@@ -15,10 +15,10 @@ export class Test_TC_OO_1_1 extends MinimalOnOffDeviceTestInstance {
         super("Test_TC_OO_1_1", "GeneralTestPicsFile.txt", storage);
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Read EventList")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_OO_2_2.ts b/chip-testing/src/suites/Test_TC_OO_2_2.ts
@@ -15,7 +15,7 @@ export class Test_TC_OO_2_2 extends MinimalOnOffDeviceTestInstance {
         super("Test_TC_OO_2_2", "GeneralTestPicsFile.txt", storage);
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("set OnOff attribute manually to on")) {
             if (this.onOffDevice !== undefined) {
                 this.onOffDevice.setOnOff(true);
@@ -31,6 +31,6 @@ export class Test_TC_OO_2_2 extends MinimalOnOffDeviceTestInstance {
                 return "n\n";
             }
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/src/suites/Test_TC_ULABEL_1_1.ts b/chip-testing/src/suites/Test_TC_ULABEL_1_1.ts
@@ -48,10 +48,10 @@ export class Test_TC_ULABEL_1_1 extends DeviceTestInstance {
         return this.commissioningServer;
     }
 
-    override async handleUserprompt(userPrompt: string, testDescription: string) {
+    override async handleUserprompt(testDescription: string, userPrompt: string) {
         if (testDescription.includes("Read the global attribute")) {
             return "y\n";
         }
-        return super.handleUserprompt(userPrompt, testDescription);
+        return super.handleUserprompt(testDescription, userPrompt);
     }
 }
diff --git a/chip-testing/test/ChipTest.ts b/chip-testing/test/ChipTest.ts
@@ -1,5 +1,6 @@
 import { Network } from "@project-chip/matter.js/net";
 import { StorageBackendMemory } from "@project-chip/matter.js/storage";
+import * as assert from "assert";
 import { spawn } from "child_process";
 import { promises as fs } from "fs";
 import { DeviceTestInstance } from "../src/DeviceTestInstance";
@@ -34,13 +35,13 @@ async function executeProcess(
             setImmediate(() => {
                 const string = data.toString();
                 // TODO: Optimize to make sure we always have complete lines ... so maybe we need to buffer
-                //       an uncomplete "last line" or such
+                //       an incomplete "last line" or such
                 const commandIndex = string.indexOf("@ChipTestRunner: Command: '");
                 if (commandIndex !== -1 && commandCallback !== undefined) {
                     const commandEndIndex = string.indexOf("'", commandIndex + 27);
                     const command = string.substring(commandIndex + 27, commandEndIndex);
                     const params = string.substring(commandEndIndex + 2, string.indexOf("\n", commandEndIndex)).trim();
-                    void commandCallback(command, params.split(",")); // Verify Params when we get some real
+                    commandCallback(command, params.split(",")).catch(error => console.log(error)); // Verify Params when we get some real
                 }
                 if (string.includes("TOO")) {
                     const lines = string.match(/^(.*[[:]TOO[\]:].*)$/gm);
@@ -52,9 +53,9 @@ async function executeProcess(
                                 lines[userPromptLineIndex - 1] ?? testResults[testResults.length - 1] ?? "";
                             const userPrompt = userPromptLine.substring(userPromptLine.indexOf("USER_PROMPT: ") + 13);
                             process.stdout.write(`====> Chip test Runner: Detected TH USER_PROMPT: ${userPrompt}\n`);
-                            userPromptCallback(userPrompt, userPromptPreviousLine)
+                            userPromptCallback(userPromptPreviousLine, userPrompt)
                                 .then(answer => proc.stdin?.write(answer)) // We acknowledge the TH reads as checked
-                                .catch(e => console.log(e));
+                                .catch(error => console.log(error));
                         }
                         testResults.push(...lines);
                     }
@@ -182,23 +183,27 @@ describe("Chip-Tool-Tests", () => {
             const storage = new StorageBackendMemory();
             const testInstance = new suite(storage) as DeviceTestInstance;
 
-            it(`"${suiteName}": Setup test instance`, async () => await testInstance.setup()).timeout(5000);
+            it(`"${suiteName}": Setup test instance`, async () =>
+                await assert.doesNotReject(async () => await testInstance.setup())).timeout(5000);
 
             it(`"${suiteName}": Start chip-tool and test instance for initial pairing`, async () =>
-                await pairWithChipTool(async () => await testInstance.start())).timeout(30000);
+                await assert.doesNotReject(
+                    async () => await pairWithChipTool(async () => await testInstance.start()),
+                )).timeout(30000);
 
             it(`"${suiteName}": Execute tests`, async () =>
-                await executeChipToolTest(
-                    testInstance.testName,
-                    testInstance.PICSConfig,
-                    (command, params) => testInstance.handleCommand(command, params),
-                    async (testDescription, userPrompt) => testInstance.handleUserprompt(testDescription, userPrompt),
+                await assert.doesNotReject(
+                    async () =>
+                        await executeChipToolTest(
+                            testInstance.testName,
+                            testInstance.PICSConfig,
+                            (command, params) => testInstance.handleCommand(command, params),
+                            async (testDescription, userPrompt) =>
+                                testInstance.handleUserprompt(testDescription, userPrompt),
+                        ),
                 )).timeout(suiteName in LongRunningTests ? 1200000 : 120000);
 
-            after(async () => {
-                await testInstance.stop();
-                //await new Promise(resolve => setTimeout(resolve, 2000)); //Add a short delay
-            });
+            after(async () => await assert.doesNotReject(async () => await testInstance.stop()));
         });
     }