ZTS: Use QEMU for tests on Linux and FreeBSD

This commit adds functional tests for these systems: - AlmaLinux 8, AlmaLinux 9 - ArchLinux - CentOS Stream 9 - Fedora 39, Fedora 40 - Debian 11, Debian 12 - FreeBSD 13, FreeBSD 14, FreeBSD 15 - Ubuntu 22.04, Ubuntu 24.04 Workflow for each operating system: - install QEMU on the github runner - download current cloud image - start and init that image via cloud-init - install deps and poweroff system - start system and build openzfs and then poweroff again - clone the system and start 3 qemu machines for tests - use trimable virtual disks (3x 2GB) - do the functional testings in < 3h Signed-off-by: Tino Reichardt <[email protected]> Co-authored-by: Tony Hutter <[email protected]>
openzfs · Jun 17, 2024 · 702642f · 702642f
1 parent c98295e
commit 702642f
Show file tree

Hide file tree

Showing 16 changed files with 1,133 additions and 391 deletions.
diff --git a/.github/workflows/scripts/README.md b/.github/workflows/scripts/README.md
@@ -0,0 +1,10 @@
+
+Workflow for each operating system:
+ - install QEMU on the github runner
+ - download current cloud image
+ - start and init that image via cloud-init
+ - install deps and poweroff system
+ - start system and build openzfs and then poweroff again
+ - clone the system and start 4 qemu workers for the testings (4x 3GB RAM)
+ - use trimable virtual disks (3x 1GB) for each testing system
+ - do the functional testings < 3h for each os
diff --git a/.github/workflows/scripts/generate-summary.sh b/.github/workflows/scripts/generate-summary.sh
@@ -1,119 +1,195 @@
 #!/usr/bin/env bash
 
-# for runtime reasons we split functional testings into N parts
-# - use a define to check for missing tarfiles
-FUNCTIONAL_PARTS="4"
-
-ZTS_REPORT="tests/test-runner/bin/zts-report.py"
-chmod +x $ZTS_REPORT
+######################################################################
+# generate github summary page of all the testings
+# /tr 2024-06-17
+######################################################################
 
 function output() {
-  echo -e $* >> Summary.md
-}
-
-function error() {
-  output ":bangbang: $* :bangbang:\n"
+  echo -e $* >> "out-$logfile.md"
 }
 
-# this function generates the real summary
-# - expects a logfile "log" in current directory
-function generate() {
-  # we issued some error already
-  test ! -s log && return
-
-  # for overview and zts-report
-  cat log | grep '^Test' > list
-
-  # error details
-  awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }
-    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err
-
-  # summary of errors
-  if [ -s err ]; then
-    output "<pre>"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
-    output "</pre>"
-
-    # generate seperate error logfile
-    ERRLOGS=$((ERRLOGS+1))
-    errfile="err-$ERRLOGS.md"
-    echo -e "\n## $headline (debugging)\n" >> $errfile
-    echo "<details><summary>Error Listing - with dmesg and dbgmsg</summary><pre>" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "</pre></details>" >> $errfile
-  else
-    output "All tests passed :thumbsup:"
+function outfile() {
+  if [ -f $1 ]; then
+    CUR=`stat --printf="%s" "out-$logfile.md"`
+    ADD=`stat --printf="%s" "$1"`
+    X=$((CUR+ADD))
+    if [ $X -lt $((1024*1023)) ]; then
+      cat "$1" >> "out-$logfile.md"
+    else
+      logfile=$((logfile+1))
+      cat "$1" >> "out-$logfile.md"
+    fi
   fi
-
-  output "<details><summary>Full Listing</summary><pre>"
-  cat list >> Summary.md
-  output "</pre></details>"
-
-  # remove tmp files
-  rm -f err list log
 }
 
-# check tarfiles and untar
-function check_tarfile() {
-  if [ -f "$1" ]; then
-    tar xf "$1" || error "Tarfile $1 returns some error"
-  else
-    error "Tarfile $1 not found"
-  fi
+function showfile() {
+  filename="$1"
+  headline="$2"
+  echo "<details><summary>$headline</summary><pre>" > tmp
+  cat $filename >> tmp
+  echo "</pre></details>" >> tmp
+  outfile tmp
+  rm -f tmp
 }
 
-# check logfile and concatenate test results
-function check_logfile() {
-  if [ -f "$1" ]; then
-    cat "$1" >> log
-  else
-    error "Logfile $1 not found"
-  fi
+function send2github() {
+  test -f "$1" && dd if="$1" bs=1023k count=1 >> $GITHUB_STEP_SUMMARY
 }
 
-# sanity
-function summarize_s() {
-  headline="$1"
+# generate summary of one test
+function generate() {
+  VMs=3
+  ####################################################################
+  # osname.txt                       -> used for headline
+  # disk-before.txt                  -> used together with uname
+  # disk-afterwards.txt              -> used together with uname
+  # vm{1,2,3}log.txt (colored, used when current/log isn't there)
+  ####################################################################
+  # vm{1,2,3}/uname.txt              -> used once
+  # vm{1,2,3}/build-stderr.txt       -> used once
+  # vm{1,2,3}/dmesg-prerun.txt       -> used once
+  # vm{1,2,3}/dmesg-module-load.txt  -> used once
+  # vm{1,2,3}/console.txt            -> all 3 used
+  ####################################################################
+  # vm{1,2,3}/current/log     -> if not there, kernel panic loading
+  # vm{1,2,3}/current/results -> if not there, kernel panic testings
+  # vm{1,2,3}/exitcode.txt
+  ####################################################################
+
+  # headline of this summary
   output "\n## $headline\n"
-  rm -rf testfiles
-  check_tarfile "$2/sanity.tar"
-  check_logfile "testfiles/log"
-  generate
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/uname.txt ]; then
+      output "<pre>"
+      outfile vm$i/uname.txt
+      output "\nVM disk usage before:"
+      outfile disk-afterwards.txt
+      output "\nand afterwards:"
+      outfile disk-before.txt
+      output "</pre>"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/build-stderr.txt ]; then
+      showfile "vm$i/build-stderr.txt" "Module build (stderr output)"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/dmesg-prerun.txt ]; then
+      showfile "vm$i/dmesg-prerun.txt" "Dmesg output - before tests"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/dmesg-module-load.txt ]; then
+      showfile "vm$i/dmesg-module-load.txt" "Dmesg output - module loading"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    log="vm$i/current/log"
+    if [ ! -f $log ]; then
+      output ":exclamation: Logfile of vm$i tests is missing :exclamation:"
+
+      # some out may be generated
+      if [ -s vm${i}log.txt ]; then
+        cat vm${i}log.txt | \
+          sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log"
+        showfile "vm${i}log" "Generated tests output of vm$i"
+      fi
+
+      # output the console contents and continue with next vm
+      if [ -s "vm$i/console.txt" ]; then
+        cat "vm$i/console.txt" | \
+          sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log"
+        showfile "vm${i}log" "Serial console output of vm$i"
+      fi
+
+      rm -f "vm${i}log"
+      continue
+    fi
+
+    cat $log | grep '^Test[: ]' > tests.txt
+    results="vm$i/current/results"
+    if [ ! -s "$results" ]; then
+      output ":exclamation: Results file of vm$i tests is missing :exclamation:"
+      # generate results file from log
+      ./zts-report.py --no-maybes ./tests.txt > $results
+      # Running Time:	01:30:09
+      # Running Time:	not finished!!
+      echo -e "\nRunning Time:\tKernel panic!" >> $results
+    fi
+    cat $results | awk '/Results Summary/ { show=1; print; next; } show' > summary.txt
+    runtime=`cat $results | grep '^Running Time:' | cut -f2`
+
+    awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } \
+      /\[SKIP\]|\[PASS\]/{ show=0; } show' $log > debug.txt
+
+    output "\n### Tests on vm$i ($runtime)\n\n"
+
+    if [ -s summary.txt ]; then
+      showfile "summary.txt" "Summary of all tests"
+    fi
+
+    if [ -s "vm$i/console.txt" ]; then
+      showfile "vm$i/console.txt" "Serial console output"
+    fi
+
+    if [ -s tests.txt ]; then
+      showfile "tests.txt" "List of all tests"
+    fi
+
+    MAX="300"
+    if [ -s debug.txt ]; then
+      S=`stat --printf="%s" "debug.txt"`
+      if [ $S -gt $((1024*$MAX)) ]; then
+        dd if=debug.txt of=debug.txt2 count=$MAX bs=1024 2>/dev/null
+        mv -f debug.txt2 debug.txt
+        echo "..." >> debug.txt
+        echo "!!! THIS FILE IS BIGGER !!!" >> debug.txt
+        echo "Please download the zip archiv for full content!" >> debug.txt
+      fi
+      showfile "debug.txt" "Debug list for failed tests (vm$i, $runtime)"
+    fi
+  done
 }
 
-# functional
-function summarize_f() {
-  headline="$1"
-  output "\n## $headline\n"
-  rm -rf testfiles
-  for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2-part$i/part$i.tar"
-    check_tarfile "$tarfile"
-    check_logfile "testfiles/log"
+# functional tests via qemu
+function summarize() {
+  for tarfile in Logs-functional-*/qemu-*.tar; do
+    rm -rf vm* *.txt
+    tar xf "$tarfile"
+    osname=`cat osname.txt`
+    headline="Functional Tests: $osname"
+    generate
   done
-  generate
 }
 
 # https://docs.github.com/en/[email protected]/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits
 # Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB.
 # [ ] can not show all error findings here
 # [x] split files into smaller ones and create additional steps
 
-ERRLOGS=0
-if [ ! -f Summary/Summary.md ]; then
-  # first call, we do the default summary (~500k)
-  echo -n > Summary.md
-  summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity
-  summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity
-  summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional
-  summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional
-
-  cat Summary.md >> $GITHUB_STEP_SUMMARY
-  mkdir -p Summary
-  mv *.md Summary
+# first call, generate all summaries
+if [ ! -f out-0.md ]; then
+  # create ./zts-report.py for generate()
+  TEMPLATE="tests/test-runner/bin/zts-report.py.in"
+  cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py
+  chmod +x ./zts-report.py
+
+  logfile="0"
+  summarize
+  send2github out-0.md
 else
-  # here we get, when errors where returned in first call
-  test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY
+  send2github out-$1.md
 fi
 
 exit 0
diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup the action runner to start some qemu instance
+######################################################################
+
+set -eu
+
+# docker isn't needed, free some memory
+sudo systemd-run --wait docker system prune --force --all --volumes
+sudo systemctl stop docker.socket
+sudo apt-get remove docker-ce-cli docker-ce podman
+
+# remove unneeded things
+sudo apt-get remove google-chrome-stable snapd
+
+# install needed packages
+sudo apt-get update
+sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \
+  virt-manager linux-modules-extra-`uname -r`
+
+# remove unused software
+df -h /
+sudo systemd-run --wait rm -rf \
+  /opt/* \
+  /usr/local/* \
+  /usr/share/az* \
+  /usr/share/dotnet \
+  /usr/share/gradle* \
+  /usr/share/miniconda \
+  /usr/share/swift \
+  /var/lib/gems \
+  /var/lib/mysql \
+  /var/lib/snapd
+
+# disk usage afterwards
+sudo df -h /
+sudo df -h /mnt
+sudo fstrim -a
+
+# generate ssh keys
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""