From b04c13c7617fbd8b47b8d50033b631474aa39539 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 21 Jun 2024 10:18:07 -0700 Subject: [PATCH] Parameterize more tests (#1364) This updates more of the tests in the tools directory to honor both `REGION_ROOT ` and `WORK_ROOT` to help avoid conflicts and give tests a place to store region files. --- tools/test_live_repair.sh | 28 +++++++++++------ tools/test_mem.sh | 5 ++- tools/test_perf.sh | 1 + tools/test_replay.sh | 13 +++++--- tools/test_restart_repair.sh | 60 ++++++++++++++++++------------------ 5 files changed, 62 insertions(+), 45 deletions(-) diff --git a/tools/test_live_repair.sh b/tools/test_live_repair.sh index f57dde390..1df780ea5 100755 --- a/tools/test_live_repair.sh +++ b/tools/test_live_repair.sh @@ -20,8 +20,15 @@ function ctrl_c() { ${dsc} cmd shutdown } -loop_log=/tmp/test_live_repair_summary.log -test_log=/tmp/test_live_repair.log +REGION_ROOT=${REGION_ROOT:-/var/tmp/test_live_repair} +mkdir -p "$REGION_ROOT" + +# Location of logs and working files +WORK_ROOT=${WORK_ROOT:-/tmp} +mkdir -p "$WORK_ROOT" + +loop_log="$WORK_ROOT"/test_live_repair_summary.log +test_log="$WORK_ROOT"/test_live_repair.log ROOT=$(cd "$(dirname "$0")/.." && pwd) export BINDIR=${BINDIR:-$ROOT/target/debug} @@ -52,12 +59,13 @@ while getopts 'l:' opt; do esac done -echo "" > ${loop_log} -echo "" > ${test_log} -echo "starting $(date)" | tee ${loop_log} +echo "" > "$loop_log" +echo "" > "$test_log" +echo "starting $(date)" | tee "$loop_log" echo "Tail $test_log for test output" if ! ${dsc} create --cleanup \ + --region-dir "$REGION_ROOT" \ --region-count 4 \ --ds-bin "$downstairs" \ --extent-size 4000 \ @@ -66,6 +74,7 @@ if ! ${dsc} create --cleanup \ exit 1 fi ${dsc} start --ds-bin "$downstairs" \ + --region-dir "$REGION_ROOT" \ --region-count 4 >> "$test_log" 2>&1 & dsc_pid=$! sleep 5 @@ -85,6 +94,7 @@ if ! "$crucible_test" fill "${args[@]}" -q -g "$gen"\ --verify-out alan --retry-activate >> "$test_log" 2>&1 ; then echo Failed on initial verify seed, check "$test_log" ${dsc} cmd shutdown + exit 1 fi (( gen += 1 )) @@ -106,7 +116,7 @@ while [[ $count -le $loops ]]; do (( err += 1 )) duration=$SECONDS printf "[%03d] Error $result after %d:%02d\n" "$count" \ - $((duration / 60)) $((duration % 60)) | tee -a ${loop_log} + $((duration / 60)) $((duration % 60)) | tee -a "$loop_log" mv "$test_log" "$test_log".lastfail break fi @@ -121,7 +131,7 @@ last_run_seconds:%d\n" \ $((duration / 60)) $((duration % 60)) \ $((ave / 60)) $((ave % 60)) \ $((total / 60)) $((total % 60)) \ - "$err" $duration | tee -a ${loop_log} + "$err" $duration | tee -a "$loop_log" (( count += 1 )) done @@ -129,12 +139,12 @@ ${dsc} cmd shutdown wait "$dsc_pid" sleep 4 -echo "Final results:" | tee -a ${loop_log} +echo "Final results:" | tee -a "$loop_log" printf "[%03d] %d:%02d ave:%d:%02d total:%d:%02d errors:%d last_run_seconds:%d\n" \ "$count" \ $((duration / 60)) $((duration % 60)) \ $((ave / 60)) $((ave % 60)) \ $((total / 60)) $((total % 60)) \ - "$err" $duration | tee -a ${loop_log} + "$err" $duration | tee -a "$loop_log" echo "$(date) Test ends with $err" >> "$test_log" 2>&1 exit "$err" diff --git a/tools/test_mem.sh b/tools/test_mem.sh index b9e9dc33e..41f7c164d 100755 --- a/tools/test_mem.sh +++ b/tools/test_mem.sh @@ -80,7 +80,10 @@ function show_mem_summary() { } -test_mem_log="/tmp/test_mem_log.txt" +# Location of logs and working files +WORK_ROOT=${WORK_ROOT:-/tmp} +mkdir -p "$WORK_ROOT" +test_mem_log="$WORK_ROOT/test_mem_log.txt" # Create a region with the given extent_size ($1) and extent_count ($2) # Once created, write to every block in the region, then display memory usage. function mem_test() { diff --git a/tools/test_perf.sh b/tools/test_perf.sh index a4d1e1632..e463e60ac 100755 --- a/tools/test_perf.sh +++ b/tools/test_perf.sh @@ -31,6 +31,7 @@ count=30000 prefill=0 read_loops=2 write_loops=2 +REGION_ROOT=${REGION_ROOT:-/var/tmp/test_perf} region_dir="/var/tmp/dsc" while getopts 'b:c:g:hfr:w:' opt; do diff --git a/tools/test_replay.sh b/tools/test_replay.sh index cdda17991..e039f3a11 100755 --- a/tools/test_replay.sh +++ b/tools/test_replay.sh @@ -14,7 +14,10 @@ function ctrl_c() { ${dsc} cmd shutdown } -test_log=/tmp/test_replay.log +WORK_ROOT=${WORK_ROOT:-/tmp} +mkdir -p "$WORK_ROOT" + +test_log="$WORK_ROOT/test_replay.log" ROOT=$(cd "$(dirname "$0")/.." && pwd) export BINDIR=${BINDIR:-$ROOT/target/debug} @@ -45,8 +48,8 @@ while getopts 'l:' opt; do esac done -echo "" > ${test_log} -echo "starting $(date)" | tee ${test_log} +echo "" > "$test_log" +echo "starting $(date)" | tee "$test_log" echo "Tail $test_log for test output" echo "Creating downstairs regions" | tee -a "$test_log" @@ -90,11 +93,11 @@ result=$? duration=$SECONDS if [[ $result -ne 0 ]]; then printf "Error $result after %d:%02d\n" \ - $((duration / 60)) $((duration % 60)) | tee -a ${test_log} + $((duration / 60)) $((duration % 60)) | tee -a "$test_log" else (( gen += 1 )) printf "Replays:%d time: %d:%02d\n" \ - "$loops" $((duration / 60)) $((duration % 60)) | tee -a ${test_log} + "$loops" $((duration / 60)) $((duration % 60)) | tee -a "$test_log" echo "Do final verify" | tee -a "$test_log" if ! "$crucible_test" verify "${args[@]}" -q -g "$gen"\ diff --git a/tools/test_restart_repair.sh b/tools/test_restart_repair.sh index 60a6b6e97..21a81c863 100755 --- a/tools/test_restart_repair.sh +++ b/tools/test_restart_repair.sh @@ -90,27 +90,28 @@ while getopts 'l:' opt; do esac done -export loop_log=/tmp/repair_restart.log -export test_log=/tmp/repair_restart_test.log -export dsc_log=/tmp/repair_restart_dsc.log -export region_dir="./var" - -echo "" > ${loop_log} -echo "starting $(date)" | tee ${loop_log} -echo "" > ${test_log} +WORK_ROOT=${WORK_ROOT:-/tmp} +export loop_log="$WORK_ROOT/repair_restart.log" +export test_log="$WORK_ROOT/repair_restart_test.log" +export dsc_log="$WORK_ROOT/repair_restart_dsc.log" +REGION_ROOT=${REGION_ROOT:-/var/tmp/test_restart_repair} + +echo "" > "$loop_log" +echo "starting $(date)" | tee "$loop_log" +echo "" > "$test_log" echo "Tail $test_log for test output" echo "Tail $loop_log for summary output" echo "Tail $dsc_log for dsc outout" echo "Create a new region to test" | tee -a "${loop_log}" ulimit -n 65536 -if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 --extent-size 5120 --region-dir "$region_dir"; then - echo "Failed to create region at $region_dir" +if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 --extent-size 5120 --region-dir "$REGION_ROOT"; then + echo "Failed to create region at $REGION_ROOT" exit 1 fi echo "Starting the downstairs" | tee -a "${loop_log}" -"$dsc" start --ds-bin "$cds" --region-dir "$region_dir" >> ${dsc_log} 2>&1 & +"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" >> "$dsc_log" 2>&1 & dsc_pid=$! # Sleep 5 to give the downstairs time to get going. sleep 5 @@ -158,10 +159,10 @@ stop_all_downstairs sleep 7 # Create the "old" region files -rm -rf "$region_dir"/8810.old "$region_dir"/8820.old "$region_dir"/8830.old -cp -R "$region_dir"/8810 "$region_dir"/8810.old || ctrl_c -cp -R "$region_dir"/8820 "$region_dir"/8820.old || ctrl_c -cp -R "$region_dir"/8830 "$region_dir"/8830.old || ctrl_c +rm -rf "$REGION_ROOT"/8810.old "$REGION_ROOT"/8820.old "$REGION_ROOT"/8830.old +cp -R "$REGION_ROOT"/8810 "$REGION_ROOT"/8810.old || ctrl_c +cp -R "$REGION_ROOT"/8820 "$REGION_ROOT"/8820.old || ctrl_c +cp -R "$REGION_ROOT"/8830 "$REGION_ROOT"/8830.old || ctrl_c # Bring the downstairs back online. echo "$(date) Bring downstairs back online" | tee -a "$loop_log" @@ -182,7 +183,7 @@ fi echo "Fill completed, wait for downstairs to start restarting" >> "$test_log" duration=$SECONDS printf "Initial fill and verify took: %d:%02d \n" \ - $((duration / 60)) $((duration % 60)) | tee -a ${loop_log} + $((duration / 60)) $((duration % 60)) | tee -a "$loop_log" # Now run the repair loop count=1 @@ -202,19 +203,19 @@ while [[ $count -le $loops ]]; do echo "$(date) move regions" >> "$test_log" choice=$((RANDOM % 3)) if [[ $choice -eq 0 ]]; then - rm -rf "$region_dir"/8810 - cp -R "$region_dir"/8810.old "$region_dir"/8810 + rm -rf "$REGION_ROOT"/8810 + cp -R "$REGION_ROOT"/8810.old "$REGION_ROOT"/8810 elif [[ $choice -eq 1 ]]; then - rm -rf "$region_dir"/8820 - cp -R "$region_dir"/8820.old "$region_dir"/8820 + rm -rf "$REGION_ROOT"/8820 + cp -R "$REGION_ROOT"/8820.old "$REGION_ROOT"/8820 else - rm -rf "$region_dir"/8830 - cp -R "$region_dir"/8830.old "$region_dir"/8830 + rm -rf "$REGION_ROOT"/8830 + cp -R "$REGION_ROOT"/8830.old "$REGION_ROOT"/8830 fi echo "$(date) regions moved, current dump outputs:" >> "$test_log" - $cds dump --no-color -d "$region_dir"/8810 \ - -d "$region_dir"/8820 \ - -d "$region_dir"/8830 >> "$test_log" 2>&1 + $cds dump --no-color -d "$REGION_ROOT"/8810 \ + -d "$REGION_ROOT"/8820 \ + -d "$REGION_ROOT"/8830 >> "$test_log" 2>&1 echo "$(date) resume downstairs" >> "$test_log" bring_all_downstairs_online @@ -236,7 +237,7 @@ while [[ $count -le $loops ]]; do (( err += 1 )) duration=$SECONDS printf "[%03d] Error $result in one test after %d:%02d\n" "$count" \ - $((duration / 60)) $((duration % 60)) | tee -a ${loop_log} + $((duration / 60)) $((duration % 60)) | tee -a "$loop_log" mv "$test_log" "$test_log".lastfail break fi @@ -249,7 +250,7 @@ while [[ $count -le $loops ]]; do printf "[%03d/%03d] %d:%02d ave:%d:%02d total:%d:%02d errors:%d \ last_run_seconds:%d\n" "$count" "$loops" $((duration / 60)) $((duration % 60)) \ $((ave / 60)) $((ave % 60)) $((total / 60)) $((total % 60)) \ -"$err" $duration | tee -a ${loop_log} +"$err" $duration | tee -a "$loop_log" (( count += 1 )) done @@ -258,11 +259,10 @@ if [[ -n "$dsc_pid" ]]; then wait "$dsc_pid" fi -echo "Final results $(date):" | tee -a ${loop_log} +echo "Final results $(date):" | tee -a "$loop_log" printf "[%03d] %d:%02d ave:%d:%02d total:%d:%02d errors:%d last_run_seconds:%d\n" \ "$count" $((duration / 60)) $((duration % 60)) \ $((ave / 60)) $((ave % 60)) \ $((total / 60)) $((total % 60)) \ - "$err" $duration | tee -a ${loop_log} + "$err" $duration | tee -a "$loop_log" exit "$err" -