Refactor tests: each test is a file (#117)

.github/workflows/test.yaml

@@ -31,7 +31,7 @@ jobs:
           cd ..
           cp -r try ~
           cd ~/try
-          bash ./test/run_tests.sh
+          scripts/run_tests.sh
 
       - name: Upload script
         uses: actions/upload-artifact@v2
@@ -107,7 +107,6 @@ jobs:
         with:
           ignore_paths: >-
             completions
-            test
 
   prerelease:
     needs:

Vagrantfile

@@ -10,11 +10,11 @@ Vagrant.configure("2") do |config|
     debian.vm.box = "debian/testing64"
     debian.vm.provision "file", source: "./", destination: "/home/vagrant/try"
     debian.vm.provision "shell", privileged: false, inline: "
-      sudo apt update
-      sudo apt install -y git expect
+      sudo apt-get update
+      sudo apt-get install -y git expect
       sudo chown -R vagrant:vagrant try
       cd try
-      bash test/run_tests.sh
+      scripts/run_tests.sh
     "
   end
 
@@ -23,8 +23,8 @@ Vagrant.configure("2") do |config|
     debianrustup.vm.box = "debian/testing64"
     debianrustup.vm.provision "file", source: "./", destination: "/home/vagrant/try"
     debianrustup.vm.provision "shell", privileged: false, inline: "
-      sudo apt update
-      sudo apt install -y curl
+      sudo apt-get update
+      sudo apt-get install -y curl
       sudo chown -R vagrant:vagrant try
       cd try
       mkdir rustup
@@ -38,8 +38,8 @@ Vagrant.configure("2") do |config|
     debianlvm.vm.box = "debian/testing64"
     debianlvm.vm.provision "file", source: "./", destination: "/home/vagrant/try"
     debianlvm.vm.provision "shell", privileged: false, inline: "
-      sudo apt update
-      sudo apt install -y git expect lvm2 mergerfs
+      sudo apt-get update
+      sudo apt-get install -y git expect lvm2 mergerfs
 
       # Create an image for the lvm disk
       sudo fallocate -l 2G /root/lvm_disk.img
@@ -65,7 +65,7 @@ Vagrant.configure("2") do |config|
       sudo chown -R vagrant:vagrant /mnt/lv0/try
 
       cd /mnt/lv0/try
-      bash test/run_tests.sh
+      scripts/run_tests.sh
     "
   end
 
@@ -77,7 +77,7 @@ Vagrant.configure("2") do |config|
       sudo yum install -y git expect
       sudo chown -R vagrant:vagrant try
       cd try
-      TRY_TOP=$(pwd) bash test/run_tests.sh
+      TRY_TOP=$(pwd) scripts/run_tests.sh
     "
   end
   #
@@ -89,7 +89,7 @@ Vagrant.configure("2") do |config|
       sudo yum install -y git expect
       sudo chown -R vagrant:vagrant try
       cd try
-      TRY_TOP=$(pwd) bash test/run_tests.sh
+      TRY_TOP=$(pwd) scripts/run_tests.sh
     "
   end
 end

scripts/README.md

@@ -0,0 +1,5 @@
+This directory holds utility scripts used in development of `try`.
+
+The `lint.sh` script runs basic file linting.
+
+The `run_tests.sh` script runs the test suite.

scripts/lint.sh

@@ -129,7 +129,7 @@ done
 
 if [ "$NUM_WARNINGS" -eq 0 ]
 then
-    printf "✓ \033[32;1mLINT CHECK PASSED\033[0m\n"
+    printf "\033[32;1m✓ LINT CHECK PASSED\033[0m\n"
 else
     printf "\n❌ \033[31;1mLINT CHECK FAILED (%d warning%s)\033[0m\n" "$NUM_WARNINGS" "$(plural "$NUM_WARNINGS")"
     exit 1

scripts/run_tests.sh

@@ -0,0 +1,94 @@
+#!/bin/sh
+
+: "${TRY_TOP=$(git rev-parse --show-toplevel --show-superproject-working-tree)}"
+TEST_DIR="$TRY_TOP/test"
+
+# set the DEBUG env variable to see detailed output
+DEBUG=${DEBUG:-0}
+
+# results saved here; clear out previous results
+OUTPUT_DIR="$TRY_TOP/test/results"
+rm -rf "$OUTPUT_DIR"
+mkdir -p "$OUTPUT_DIR"
+touch "$OUTPUT_DIR/result_status"
+
+TOTAL_TESTS=0
+PASSED_TESTS=0
+FAILING_TESTS=""
+
+run_test()
+{
+    test="$1"
+    name="$(basename "$test")"
+
+    if ! [ -f "$test" ]
+    then
+        echo "Test '$name' does not exist (at path $test)."
+        return 1
+    elif ! [ -x "$test" ]
+    then
+        echo "Test '$name' is not executable (at path $test)."
+        return 1
+    fi
+
+    # check if we can read from /run dir
+    ls /run/systemd >/dev/null || {
+        echo "Cannot read from /run/systemd, aborting test '$name'." >&2
+        return 1
+    }
+
+    echo
+    printf "Running %s..." "$name"
+
+    # actually run the test
+    "$test"
+    ec="$?"
+
+    : $((TOTAL_TESTS += 1))
+    if [ "$ec" -eq 0 ]; then
+        : $((PASSED_TESTS += 1))
+        printf '\t\t\t'
+        echo "Test $name passed" >>"$OUTPUT_DIR/result_status"
+        printf '\tOK\n'
+    else
+        FAILING_TESTS="$FAILING_TESTS $name"
+        printf " non-zero exit status (%d)" "$ec"
+        echo "Test $name failed" >>"$OUTPUT_DIR/result_status"
+        printf '\t\tFAIL\n'
+    fi
+}
+
+pats="$(mktemp)"
+if [ "$#" -eq 0 ]
+then
+    # if no patterns are specified
+    echo '.*' >"$pats"
+else
+    printf "%s\n" "$@" >>"$pats"
+fi
+
+TESTS="$(find "$TEST_DIR" -type f -executable -name '*.sh' | grep -f "$pats")"
+rm "$pats"
+
+echo "=================| Try Tests |==================="
+echo "Test directory:           $WORKING_DIR"
+echo "Results saved at:         $OUTPUT_DIR"
+echo
+echo "Running $(echo "$TESTS" | wc -l) tests"
+echo "================================================="
+
+for test in $TESTS
+do
+    run_test "$test"
+done
+
+echo
+echo "====================| Test Summary |===================="
+echo "Failing tests:${FAILING_TESTS}"
+echo "Summary: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed."
+echo "========================================================"
+
+if [ $PASSED_TESTS -ne $TOTAL_TESTS ]
+then
+    exit 1
+fi

test/README.md

@@ -0,0 +1,17 @@
+This directory holds tests and testing resources.
+
+`scripts/run_tests.sh` will run all of the tests in this directory
+(and subdirectories); to be a test, a file must be:
+
+  - a regular file
+  - executable
+  - end in `.sh`
+
+You can select a specific test by running `scripts/run_tests.sh a b`;
+this will run tests that pass `a` _or_ `b` as grep patterns.
+
+Some files are named `.sh.skip`---these will always be skipped. Rename
+them if you need to run them.
+
+Tests have a pretty standard preamble that should handle basic cleanup
+functions.

test/command_substitution.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}"
+TRY="$TRY_TOP/try"
+
+cleanup() {
+    cd /
+
+    if [ -d "$try_workspace" ]
+    then
+        rm -rf "$try_workspace" >/dev/null 2>&1
+    fi
+
+    if [ -f "$expected" ]
+    then
+        rm "$expected"
+    fi
+}
+
+trap 'cleanup' EXIT
+
+try_workspace="$(mktemp -d)"
+cd "$try_workspace" || return 9
+
+# Set up expected output
+expected="$(mktemp)"
+pwd >"$expected"
+
+cat >script.sh <<"EOF"
+    mypwd="$(pwd)"
+    echo "$mypwd"
+EOF
+
+"$TRY" sh script.sh >out.txt || return 1
+
+diff -q "$expected" out.txt

test/dev_urandom.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}"
+TRY="$TRY_TOP/try"
+
+cleanup() {
+    cd /
+
+    if [ -d "$try_workspace" ]
+    then
+        rm -rf "$try_workspace" >/dev/null 2>&1
+    fi
+}
+
+trap 'cleanup' EXIT
+
+try_workspace="$(mktemp -d)"
+cd "$try_workspace" || return 9
+
+# Ignore changes to foo
+"$TRY" -y "head -c 5 /dev/urandom >target" || return 1
+[ -s target ] || return 2

test/empty_summary.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}"
+TRY="$TRY_TOP/try"
+
+cleanup() {
+    cd /
+
+    if [ -d "$try_workspace" ]
+    then
+        rm -rf "$try_workspace" >/dev/null 2>&1
+    fi
+
+    if [ -d "$try_example_dir" ]
+    then
+        rm -rf "$try_example_dir"
+    fi
+}
+
+trap 'cleanup' EXIT
+
+try_workspace="$(mktemp -d)"
+cd "$try_workspace" || return 9
+
+try_example_dir="$(mktemp -d)"
+"$TRY" -D "$try_example_dir" -- echo hi >/dev/null || return 1
+"$TRY" summary "$try_example_dir" >summary.out
+
+# an empty summary returns exit status 1
+[ "$?" -eq 1 ] || return 2
+
+# We want to return true if the following line is not found!
+! grep -q -e "Changes detected in the following files:" summary.out

test/exit_status.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}"
+TRY="$TRY_TOP/try"
+
+cleanup() {
+    cd /
+
+    if [ -d "$try_workspace" ]
+    then
+        rm -rf "$try_workspace" >/dev/null 2>&1
+    fi
+}
+
+trap 'cleanup' EXIT
+
+try_workspace="$(mktemp -d)"
+cd "$try_workspace" || return 9
+
+"$TRY" exit 3
+
+[ "$?" -eq 3 ]

test/explore.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}"
+TRY="$TRY_TOP/try"
+
+cleanup() {
+    cd /
+
+    if [ -d "$try_workspace" ]
+    then
+        rm -rf "$try_workspace" >/dev/null 2>&1
+    fi
+}
+
+trap 'cleanup' EXIT
+
+try_workspace="$(mktemp -d)"
+cd "$try_workspace" || return 9
+
+SHELL="/bin/bash --norc"
+export SHELL
+PS1="# "
+export PS1
+
+echo hi >expected.out
+
+cat >explore.exp <<EOF
+#!/usr/bin/expect
+
+set timeout 3
+
+spawn "$TRY" explore
+expect {
+    # Ignore the warnings
+    "Warning*" {
+        exp_continue
+    }
+    # When we get the prompt, send the command
+    "#*" {
+        send -- "echo hi>test.txt\r"
+    }
+  }
+expect "#"
+# Send exit
+send \x04
+
+# Ignore all output and just send a y at the end
+expect ""
+expect "Commit*"
+send -- "y\r"
+expect eof
+EOF
+
+# Debug using the -d flag
+expect explore.exp >/dev/null || return 1
+
+diff -q expected.out test.txt

test/fail.sh.skip

@@ -0,0 +1,5 @@
+#!/bin/sh
+#
+# rename this to fail.sh to force a CI failure (e.g., when testing CI changes)
+
+exit 1