From c3401657d85cf3042ba75ad8b918ec1e447cd4a4 Mon Sep 17 00:00:00 2001
From: Wesley Hershberger <wesley.hershberger@canonical.com>
Date: Mon, 17 Jun 2024 15:58:58 -0500
Subject: [PATCH] example/test: Add cluster recovery test

They're not pretty

Signed-off-by: Wesley Hershberger <wesley.hershberger@canonical.com>
---
 example/test/main.sh | 47 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/example/test/main.sh b/example/test/main.sh
index 1c92b6b5..7a2d073c 100755
--- a/example/test/main.sh
+++ b/example/test/main.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 cluster_flags=()
 
 if [ -n "${DEBUG:-}" ]; then
@@ -48,6 +50,14 @@ for member in "${members[@]:1}"; do
   indx=$((indx + 1))
 done
 
+# dqlite takes a while to form the cluster and assign roles to each node, and
+# microcluster takes a while to update the core_cluster_members table
+while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do
+  sleep 2
+done
+
+microctl --state-dir "${test_dir}/c1" cluster list
+
 # Clean up
 if [ -n "${CLUSTER_INSPECT:-}" ]; then
   echo "Pausing to inspect... press enter when done"
@@ -58,7 +68,40 @@ for member in "${members[@]}"; do
   microctl --state-dir "${test_dir}/${member}" shutdown
 done
 
-kill 0
+# The cluster doesn't always shut down right away; this is fine since we're
+# doing recovery next
+for jobnum in {1..5}; do
+  kill -9 %"${jobnum}"
+done
 
-sleep 1
+microctl --state-dir "${test_dir}/c1" cluster list --local |
+  yq '
+    sort_by(.name) |
+    .[0].role = "voter" |
+    .[1].role = "voter" |
+    .[2].role = "spare" |
+    .[3].role = "spare" |
+    .[4].role = "spare"' |
+  microctl --state-dir "${test_dir}/c1" cluster edit
 
+cp "${test_dir}/c1/recovery_db.tar.gz" "${test_dir}/c2/"
+
+for member in c1 c2; do
+  state_dir="${test_dir}/${member}"
+  microd --state-dir "${state_dir}" "${cluster_flags[@]}" > /dev/null 2>&1 &
+done
+
+# microcluster takes a long time to update the member roles in the core_cluster_members table
+sleep 90
+
+microctl --state-dir "${test_dir}/c1" cluster list
+
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c1").role') == "voter" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c2").role') == "voter" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c3").role') == "spare" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c4").role') == "spare" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c5").role') == "spare" ]]
+
+echo "Tests passed"
+
+kill 0