From 69bf4dc6860e784127dae95842555bfeca85ea57 Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Tue, 2 Jul 2024 17:24:07 +0200 Subject: [PATCH 01/11] Improve testing flow and logging --- tests/upgrade_test.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index 0f1c2eb..2275af4 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -1,12 +1,13 @@ package tests import ( - "github.com/canonical/matter-snap-testing/utils" - "github.com/stretchr/testify/assert" "log" "os" "testing" "time" + + "github.com/canonical/matter-snap-testing/utils" + "github.com/stretchr/testify/assert" ) func TestUpgrade(t *testing.T) { @@ -49,7 +50,7 @@ func TestUpgrade(t *testing.T) { t.Run("Commission", func(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool pairing onnetwork 110 20202021 2>&1") assert.NoError(t, - os.WriteFile("chip-tool-pairing.log", []byte(stdout), 0644), + os.WriteFile(t.Name()+"-chip-tool-pairing.log", []byte(stdout), 0644), ) }) @@ -59,9 +60,10 @@ func TestUpgrade(t *testing.T) { snapRevision := utils.SnapRevision(t, chipToolSnap) log.Printf("%s installed version %s build %s\n", chipToolSnap, snapVersion, snapRevision) + start := time.Now() stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") assert.NoError(t, - os.WriteFile("chip-tool-onoff.log", []byte(stdout), 0644), + os.WriteFile(t.Name()+"-chip-tool-onoff.log", []byte(stdout), 0644), ) utils.WaitForLogMessage(t, @@ -69,11 +71,13 @@ func TestUpgrade(t *testing.T) { }) // Upgrade chip-tool to local snap or edge - if utils.LocalServiceSnap() { - utils.SnapInstallFromFile(t, utils.LocalServiceSnapPath) - } else { - utils.SnapRefresh(t, chipToolSnap, "latest/edge") - } + t.Run("Refresh snap", func(t *testing.T) { + if utils.LocalServiceSnap() { + utils.SnapInstallFromFile(t, utils.LocalServiceSnapPath) + } else { + utils.SnapRefresh(t, chipToolSnap, "latest/edge") + } + }) // Control device again t.Run("Control upgraded snap", func(t *testing.T) { @@ -81,9 +85,10 @@ func TestUpgrade(t *testing.T) { snapRevision := utils.SnapRevision(t, chipToolSnap) log.Printf("%s installed version %s build %s\n", chipToolSnap, snapVersion, snapRevision) + start := time.Now() stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") assert.NoError(t, - os.WriteFile("chip-tool-onoff.log", []byte(stdout), 0644), + os.WriteFile(t.Name()+"-chip-tool-onoff.log", []byte(stdout), 0644), ) utils.WaitForLogMessage(t, From 21de9f926db27d24cba99e35012b46d50fdc2c5c Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Wed, 3 Jul 2024 17:34:18 +0200 Subject: [PATCH 02/11] Unique naming of log files from subtests --- tests/common.go | 9 +++++++++ tests/upgrade_test.go | 31 +++++++++---------------------- tests/wifi_test.go | 18 ++++++------------ 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/tests/common.go b/tests/common.go index e3c26b0..6f99a80 100644 --- a/tests/common.go +++ b/tests/common.go @@ -1,9 +1,12 @@ package tests import ( + "os" + "strings" "testing" "github.com/canonical/matter-snap-testing/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,3 +36,9 @@ func InstallChipTool(t *testing.T) { utils.SnapConnect(t, chipToolSnap+":bluez", "") utils.SnapConnect(t, chipToolSnap+":process-control", "") } + +func writeLogFile(t *testing.T, label string, b []byte) { + assert.NoError(t, + os.WriteFile(strings.ReplaceAll(t.Name(), "/", "-")+"-"+label+".log", b, 0644), + ) +} diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index 2275af4..f90e0da 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -2,18 +2,15 @@ package tests import ( "log" - "os" "testing" "time" "github.com/canonical/matter-snap-testing/utils" - "github.com/stretchr/testify/assert" ) func TestUpgrade(t *testing.T) { start := time.Now() - // Remove snaps and logs at end of test, even if it failed t.Cleanup(func() { utils.SnapRemove(nil, allClustersSnap) utils.SnapDumpLogs(nil, start, allClustersSnap) @@ -46,32 +43,25 @@ func TestUpgrade(t *testing.T) { utils.WaitForLogMessage(t, allClustersSnap, "CHIP minimal mDNS started advertising", start) - // Pair device t.Run("Commission", func(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool pairing onnetwork 110 20202021 2>&1") - assert.NoError(t, - os.WriteFile(t.Name()+"-chip-tool-pairing.log", []byte(stdout), 0644), - ) + writeLogFile(t, "chip-tool-pairing", []byte(stdout)) }) - // Control device - t.Run("Control with stable snap", func(t *testing.T) { + t.Run("Control before upgrade", func(t *testing.T) { snapVersion := utils.SnapVersion(t, chipToolSnap) snapRevision := utils.SnapRevision(t, chipToolSnap) log.Printf("%s installed version %s build %s\n", chipToolSnap, snapVersion, snapRevision) start := time.Now() - stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") - assert.NoError(t, - os.WriteFile(t.Name()+"-chip-tool-onoff.log", []byte(stdout), 0644), - ) + stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff on 110 1 2>&1") + writeLogFile(t, "chip-tool-onoff", []byte(stdout)) utils.WaitForLogMessage(t, - allClustersSnap, "CHIP:ZCL: Toggle ep1 on/off", start) + allClustersSnap, "Toggle ep1 on/off from state 0 to 1", start) }) - // Upgrade chip-tool to local snap or edge - t.Run("Refresh snap", func(t *testing.T) { + t.Run("Upgrade snap", func(t *testing.T) { if utils.LocalServiceSnap() { utils.SnapInstallFromFile(t, utils.LocalServiceSnapPath) } else { @@ -79,20 +69,17 @@ func TestUpgrade(t *testing.T) { } }) - // Control device again t.Run("Control upgraded snap", func(t *testing.T) { snapVersion := utils.SnapVersion(t, chipToolSnap) snapRevision := utils.SnapRevision(t, chipToolSnap) log.Printf("%s installed version %s build %s\n", chipToolSnap, snapVersion, snapRevision) start := time.Now() - stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") - assert.NoError(t, - os.WriteFile(t.Name()+"-chip-tool-onoff.log", []byte(stdout), 0644), - ) + stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff off 110 1 2>&1") + writeLogFile(t, "chip-tool-onoff", []byte(stdout)) utils.WaitForLogMessage(t, - allClustersSnap, "CHIP:ZCL: Toggle ep1 on/off", start) + allClustersSnap, "Toggle ep1 on/off from state 1 to 0", start) }) } diff --git a/tests/wifi_test.go b/tests/wifi_test.go index 35bbb54..25577a0 100644 --- a/tests/wifi_test.go +++ b/tests/wifi_test.go @@ -1,12 +1,10 @@ package tests import ( - "os" "testing" "time" "github.com/canonical/matter-snap-testing/utils" - "github.com/stretchr/testify/assert" ) func TestAllClustersAppWiFi(t *testing.T) { @@ -19,11 +17,11 @@ func TestAllClustersAppWiFi(t *testing.T) { t.Cleanup(func() { utils.SnapRemove(t, allClustersSnap) - utils.SnapDumpLogs(nil, start, allClustersSnap) + utils.SnapDumpLogs(t, start, allClustersSnap) }) // Install all clusters app - utils.SnapInstallFromStore(t, allClustersSnap, utils.ServiceChannel) + utils.SnapInstallFromStore(t, allClustersSnap, "latest/edge") // Setup all clusters app utils.SnapSet(t, allClustersSnap, "args", "--wifi") @@ -37,19 +35,15 @@ func TestAllClustersAppWiFi(t *testing.T) { t.Run("Commission", func(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool pairing onnetwork 110 20202021 2>&1") - assert.NoError(t, - os.WriteFile("chip-tool-pairing.log", []byte(stdout), 0644), - ) + writeLogFile(t, "chip-tool-pairing", []byte(stdout)) }) t.Run("Control", func(t *testing.T) { - stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") - assert.NoError(t, - os.WriteFile("chip-tool-onoff.log", []byte(stdout), 0644), - ) + + writeLogFile(t, "chip-tool-onoff", []byte(stdout)) utils.WaitForLogMessage(t, - allClustersSnap, "CHIP:ZCL: Toggle ep1 on/off", start) + allClustersSnap, "Toggle ep1 on/off", start) }) } From e0d3a1e997e25e26353610c538fb7dc1660299ba Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Wed, 3 Jul 2024 17:35:40 +0200 Subject: [PATCH 03/11] Line buffer standard output to allow testing on current logs --- tests/wifi_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/wifi_test.go b/tests/wifi_test.go index 25577a0..e07a0ae 100644 --- a/tests/wifi_test.go +++ b/tests/wifi_test.go @@ -39,6 +39,8 @@ func TestAllClustersAppWiFi(t *testing.T) { }) t.Run("Control", func(t *testing.T) { + // stdbuf --output=L sets the output stream to line buffered + stdout, _, _ := utils.Exec(t, "stdbuf --output=L sudo chip-tool onoff toggle 110 1 2>&1") writeLogFile(t, "chip-tool-onoff", []byte(stdout)) From f229b90815c5aea9397311259db24004b8eaa1c1 Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Wed, 3 Jul 2024 17:54:47 +0200 Subject: [PATCH 04/11] Remove obsolete journal query for chip-tool --- tests/upgrade_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index f90e0da..1d663e3 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -14,8 +14,8 @@ func TestUpgrade(t *testing.T) { t.Cleanup(func() { utils.SnapRemove(nil, allClustersSnap) utils.SnapDumpLogs(nil, start, allClustersSnap) + utils.SnapRemove(nil, chipToolSnap) - utils.SnapDumpLogs(nil, start, chipToolSnap) }) // Start clean From f5b9cb16e20379a1c5d364ed383065eff0312344 Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Wed, 3 Jul 2024 18:14:58 +0200 Subject: [PATCH 05/11] Switch to unbuffered --- tests/wifi_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/wifi_test.go b/tests/wifi_test.go index e07a0ae..9081cee 100644 --- a/tests/wifi_test.go +++ b/tests/wifi_test.go @@ -39,10 +39,9 @@ func TestAllClustersAppWiFi(t *testing.T) { }) t.Run("Control", func(t *testing.T) { - // stdbuf --output=L sets the output stream to line buffered - stdout, _, _ := utils.Exec(t, "stdbuf --output=L sudo chip-tool onoff toggle 110 1 2>&1") - - writeLogFile(t, "chip-tool-onoff", []byte(stdout)) + // stdbuf --output=0 sets the output stream to unbuffered + stdout, _, _ := utils.Exec(t, "stdbuf --output=0 sudo chip-tool onoff toggle 110 1 2>&1") + writeLogFile(t, "chip-tool-toggle", []byte(stdout)) utils.WaitForLogMessage(t, allClustersSnap, "Toggle ep1 on/off", start) From 3f97288b219573b94e05232707298d6c64fee7f5 Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Wed, 3 Jul 2024 22:29:55 +0200 Subject: [PATCH 06/11] Assert based on cluster ID in the received request --- tests/wifi_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/wifi_test.go b/tests/wifi_test.go index 9081cee..d0587bd 100644 --- a/tests/wifi_test.go +++ b/tests/wifi_test.go @@ -39,12 +39,13 @@ func TestAllClustersAppWiFi(t *testing.T) { }) t.Run("Control", func(t *testing.T) { - // stdbuf --output=0 sets the output stream to unbuffered - stdout, _, _ := utils.Exec(t, "stdbuf --output=0 sudo chip-tool onoff toggle 110 1 2>&1") + stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") writeLogFile(t, "chip-tool-toggle", []byte(stdout)) - utils.WaitForLogMessage(t, - allClustersSnap, "Toggle ep1 on/off", start) + // 0x6 is the cluster ID for on-off + // Using cluster ID here because of a buffering issue in the log stream: + // https://github.com/canonical/chip-tool-snap/pull/69#issuecomment-2207189962 + utils.WaitForLogMessage(t, allClustersSnap, "ClusterId = 0x6", start) }) } From 8c4d5a8a43c7734e8e9fc06d8ea08308bf605d31 Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Thu, 4 Jul 2024 17:44:51 +0200 Subject: [PATCH 07/11] Use cluster ID in all tests for checking the on-off handling --- tests/common.go | 8 ++++++++ tests/upgrade_test.go | 6 ++---- tests/wifi_test.go | 5 +---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/common.go b/tests/common.go index 6f99a80..04ac88d 100644 --- a/tests/common.go +++ b/tests/common.go @@ -4,6 +4,7 @@ import ( "os" "strings" "testing" + "time" "github.com/canonical/matter-snap-testing/utils" "github.com/stretchr/testify/assert" @@ -42,3 +43,10 @@ func writeLogFile(t *testing.T, label string, b []byte) { os.WriteFile(strings.ReplaceAll(t.Name(), "/", "-")+"-"+label+".log", b, 0644), ) } + +func waitForOnOffHandingByAllClustersApp(t *testing.T, start time.Time) { + // 0x6 is the Matter Cluster ID for on-off + // Using cluster ID here because of a buffering issue in the log stream: + // https://github.com/canonical/chip-tool-snap/pull/69#issuecomment-2207189962 + utils.WaitForLogMessage(t, allClustersSnap, "ClusterId = 0x6", start) +} diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index 1d663e3..3b9b0b1 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -57,8 +57,7 @@ func TestUpgrade(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff on 110 1 2>&1") writeLogFile(t, "chip-tool-onoff", []byte(stdout)) - utils.WaitForLogMessage(t, - allClustersSnap, "Toggle ep1 on/off from state 0 to 1", start) + waitForOnOffHandingByAllClustersApp(t, start) }) t.Run("Upgrade snap", func(t *testing.T) { @@ -78,8 +77,7 @@ func TestUpgrade(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff off 110 1 2>&1") writeLogFile(t, "chip-tool-onoff", []byte(stdout)) - utils.WaitForLogMessage(t, - allClustersSnap, "Toggle ep1 on/off from state 1 to 0", start) + waitForOnOffHandingByAllClustersApp(t, start) }) } diff --git a/tests/wifi_test.go b/tests/wifi_test.go index d0587bd..ac72231 100644 --- a/tests/wifi_test.go +++ b/tests/wifi_test.go @@ -42,10 +42,7 @@ func TestAllClustersAppWiFi(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") writeLogFile(t, "chip-tool-toggle", []byte(stdout)) - // 0x6 is the cluster ID for on-off - // Using cluster ID here because of a buffering issue in the log stream: - // https://github.com/canonical/chip-tool-snap/pull/69#issuecomment-2207189962 - utils.WaitForLogMessage(t, allClustersSnap, "ClusterId = 0x6", start) + waitForOnOffHandingByAllClustersApp(t, start) }) } From 0a0808efcaeb78bf4d216742c8da27b1b16916dc Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Thu, 4 Jul 2024 22:24:38 +0200 Subject: [PATCH 08/11] Change assertion to look for cluster ID in thread tests --- tests/thread_tests/thread_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/thread_tests/thread_test.go b/tests/thread_tests/thread_test.go index 2b54743..d770d81 100644 --- a/tests/thread_tests/thread_test.go +++ b/tests/thread_tests/thread_test.go @@ -30,7 +30,10 @@ func TestAllClustersAppThread(t *testing.T) { os.WriteFile("chip-tool-thread-onoff.log", []byte(stdout), 0644), ) - remote_waitForLogMessage(t, "matter-all-clusters-app", "CHIP:ZCL: Toggle ep1 on/off", start) + // 0x6 is the Matter Cluster ID for on-off + // Using cluster ID here because of a buffering issue in the log stream: + // https://github.com/canonical/chip-tool-snap/pull/69#issuecomment-2209530275 + remote_waitForLogMessage(t, "matter-all-clusters-app", "ClusterId = 0x6", start) }) } From 483228e87735e425bf3156827d6a4629e51926ee Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Thu, 4 Jul 2024 22:25:58 +0200 Subject: [PATCH 09/11] Set apt install to not prompt for user input --- tests/thread_tests/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/thread_tests/remote.go b/tests/thread_tests/remote.go index 67c672f..859ee3d 100644 --- a/tests/thread_tests/remote.go +++ b/tests/thread_tests/remote.go @@ -119,7 +119,7 @@ func remote_deployAllClustersApp(t *testing.T) { start := time.Now().UTC() commands := []string{ - "sudo apt install bluez", + "sudo apt install -y bluez", "sudo snap remove --purge matter-all-clusters-app", "sudo snap install matter-all-clusters-app --edge", "sudo snap set matter-all-clusters-app args='--thread'", From 4e0d74a109e142a1ca9eb59938d0b562d98d67f2 Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Thu, 4 Jul 2024 22:28:04 +0200 Subject: [PATCH 10/11] Comment out bluetooth setup as it is unused --- tests/thread_tests/remote.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/thread_tests/remote.go b/tests/thread_tests/remote.go index 859ee3d..75370b3 100644 --- a/tests/thread_tests/remote.go +++ b/tests/thread_tests/remote.go @@ -119,12 +119,12 @@ func remote_deployAllClustersApp(t *testing.T) { start := time.Now().UTC() commands := []string{ - "sudo apt install -y bluez", + // "sudo apt install -y bluez", "sudo snap remove --purge matter-all-clusters-app", "sudo snap install matter-all-clusters-app --edge", "sudo snap set matter-all-clusters-app args='--thread'", "sudo snap connect matter-all-clusters-app:avahi-control", - "sudo snap connect matter-all-clusters-app:bluez", + // "sudo snap connect matter-all-clusters-app:bluez", "sudo snap connect matter-all-clusters-app:otbr-dbus-wpan0 openthread-border-router:dbus-wpan0", "sudo snap start matter-all-clusters-app", } From 5304a22a22aa8b8fddfb5e6078c1b96215a78b7a Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Thu, 4 Jul 2024 22:28:25 +0200 Subject: [PATCH 11/11] Fix typo in function name --- tests/common.go | 2 +- tests/upgrade_test.go | 4 ++-- tests/wifi_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/common.go b/tests/common.go index 04ac88d..7b65f4f 100644 --- a/tests/common.go +++ b/tests/common.go @@ -44,7 +44,7 @@ func writeLogFile(t *testing.T, label string, b []byte) { ) } -func waitForOnOffHandingByAllClustersApp(t *testing.T, start time.Time) { +func waitForOnOffHandlingByAllClustersApp(t *testing.T, start time.Time) { // 0x6 is the Matter Cluster ID for on-off // Using cluster ID here because of a buffering issue in the log stream: // https://github.com/canonical/chip-tool-snap/pull/69#issuecomment-2207189962 diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index 3b9b0b1..a008e82 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -57,7 +57,7 @@ func TestUpgrade(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff on 110 1 2>&1") writeLogFile(t, "chip-tool-onoff", []byte(stdout)) - waitForOnOffHandingByAllClustersApp(t, start) + waitForOnOffHandlingByAllClustersApp(t, start) }) t.Run("Upgrade snap", func(t *testing.T) { @@ -77,7 +77,7 @@ func TestUpgrade(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff off 110 1 2>&1") writeLogFile(t, "chip-tool-onoff", []byte(stdout)) - waitForOnOffHandingByAllClustersApp(t, start) + waitForOnOffHandlingByAllClustersApp(t, start) }) } diff --git a/tests/wifi_test.go b/tests/wifi_test.go index ac72231..5b63acc 100644 --- a/tests/wifi_test.go +++ b/tests/wifi_test.go @@ -42,7 +42,7 @@ func TestAllClustersAppWiFi(t *testing.T) { stdout, _, _ := utils.Exec(t, "sudo chip-tool onoff toggle 110 1 2>&1") writeLogFile(t, "chip-tool-toggle", []byte(stdout)) - waitForOnOffHandingByAllClustersApp(t, start) + waitForOnOffHandlingByAllClustersApp(t, start) }) }