example/test: Add cluster recovery test

They're not pretty

Signed-off-by: Wesley Hershberger <[email protected]>

example/test/main.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 cluster_flags=()
 
 if [ -n "${DEBUG:-}" ]; then
@@ -48,6 +50,14 @@ for member in "${members[@]:1}"; do
   indx=$((indx + 1))
 done
 
+# dqlite takes a while to form the cluster and assign roles to each node, and
+# microcluster takes a while to update the core_cluster_members table
+while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do
+  sleep 2
+done
+
+microctl --state-dir "${test_dir}/c1" cluster list
+
 # Clean up
 if [ -n "${CLUSTER_INSPECT:-}" ]; then
   echo "Pausing to inspect... press enter when done"
@@ -58,7 +68,40 @@ for member in "${members[@]}"; do
   microctl --state-dir "${test_dir}/${member}" shutdown
 done
 
-kill 0
+# The cluster doesn't always shut down right away; this is fine since we're
+# doing recovery next
+for jobnum in {1..5}; do
+  kill -9 %"${jobnum}"
+done
 
-sleep 1
+microctl --state-dir "${test_dir}/c1" cluster list --local |
+  yq '
+    sort_by(.name) |
+    .[0].role = "voter" |
+    .[1].role = "voter" |
+    .[2].role = "spare" |
+    .[3].role = "spare" |
+    .[4].role = "spare"' |
+  microctl --state-dir "${test_dir}/c1" cluster edit
 
+cp "${test_dir}/c1/recovery_db.tar.gz" "${test_dir}/c2/"
+
+for member in c1 c2; do
+  state_dir="${test_dir}/${member}"
+  microd --state-dir "${state_dir}" "${cluster_flags[@]}" > /dev/null 2>&1 &
+done
+
+# microcluster takes a long time to update the member roles in the core_cluster_members table
+sleep 90
+
+microctl --state-dir "${test_dir}/c1" cluster list
+
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c1").role') == "voter" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c2").role') == "voter" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c3").role') == "spare" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c4").role') == "spare" ]]
+[[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c5").role') == "spare" ]]
+
+echo "Tests passed"
+
+kill 0