From 3fd75b90da8a46bba65fd3d4a4ae7ea2fc845ede Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Mon, 17 Jun 2024 16:52:58 +0200 Subject: [PATCH] ZTS: Use QEMU for tests on Linux and FreeBSD This commit adds functional tests for these systems: - AlmaLinux 8, AlmaLinux 9 - ArchLinux - CentOS Stream 9 - Fedora 39, Fedora 40 - Debian 11, Debian 12 - FreeBSD 13, FreeBSD 14, FreeBSD 15 - Ubuntu 22.04, Ubuntu 24.04 Workflow for each operating system: - install QEMU on the github runner - download current cloud image - start and init that image via cloud-init - install deps and poweroff system - start system and build openzfs and then poweroff again - clone the system and start 3 qemu machines for tests - use trimable virtual disks (3x 2GB) - do the functional testings in < 3h Signed-off-by: Tino Reichardt Co-authored-by: Tony Hutter --- .github/workflows/scripts/generate-summary.sh | 258 ++++++++++++------ .../workflows/scripts/setup-dependencies.sh | 88 ------ .github/workflows/scripts/setup-functional.sh | 24 -- .github/workflows/zfs-linux-tests.yml | 124 --------- .github/workflows/zfs-linux.yml | 64 ----- .../cli_root/zfs_share/zfs_share_005_pos.ksh | 1 + .../redacted_send/redacted_deleted.ksh | 2 + 7 files changed, 170 insertions(+), 391 deletions(-) delete mode 100755 .github/workflows/scripts/setup-dependencies.sh delete mode 100755 .github/workflows/scripts/setup-functional.sh delete mode 100644 .github/workflows/zfs-linux-tests.yml delete mode 100644 .github/workflows/zfs-linux.yml diff --git a/.github/workflows/scripts/generate-summary.sh b/.github/workflows/scripts/generate-summary.sh index b5d89208a5d8..d5e6d91cc30b 100755 --- a/.github/workflows/scripts/generate-summary.sh +++ b/.github/workflows/scripts/generate-summary.sh @@ -1,97 +1,176 @@ #!/usr/bin/env bash -# for runtime reasons we split functional testings into N parts -# - use a define to check for missing tarfiles -FUNCTIONAL_PARTS="4" - -ZTS_REPORT="tests/test-runner/bin/zts-report.py" -chmod +x $ZTS_REPORT +###################################################################### +# generate github summary page of all the testings +# /tr 2024-06-17 +###################################################################### function output() { - echo -e $* >> Summary.md -} - -function error() { - output ":bangbang: $* :bangbang:\n" + echo -e $* >> "out-$logfile.md" } -# this function generates the real summary -# - expects a logfile "log" in current directory -function generate() { - # we issued some error already - test ! -s log && return - - # for overview and zts-report - cat log | grep '^Test' > list - - # error details - awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } - /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err - - # summary of errors - if [ -s err ]; then - output "
"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
-    output "
" - - # generate seperate error logfile - ERRLOGS=$((ERRLOGS+1)) - errfile="err-$ERRLOGS.md" - echo -e "\n## $headline (debugging)\n" >> $errfile - echo "
Error Listing - with dmesg and dbgmsg
" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "
" >> $errfile - else - output "All tests passed :thumbsup:" +function outfile() { + if [ -f $1 ]; then + CUR=`stat --printf="%s" "out-$logfile.md"` + ADD=`stat --printf="%s" "$1"` + X=$((CUR+ADD)) + if [ $X -lt $((1024*1023)) ]; then + cat "$1" >> "out-$logfile.md" + else + logfile=$((logfile+1)) + cat "$1" >> "out-$logfile.md" + fi fi - - output "
Full Listing
"
-  cat list >> Summary.md
-  output "
" - - # remove tmp files - rm -f err list log } -# check tarfiles and untar -function check_tarfile() { - if [ -f "$1" ]; then - tar xf "$1" || error "Tarfile $1 returns some error" - else - error "Tarfile $1 not found" - fi +function showfile() { + filename="$1" + headline="$2" + echo "
$headline
" > tmp
+  cat $filename >> tmp
+  echo "
" >> tmp + outfile tmp + rm -f tmp } -# check logfile and concatenate test results -function check_logfile() { - if [ -f "$1" ]; then - cat "$1" >> log - else - error "Logfile $1 not found" - fi +function send2github() { + test -f "$1" && dd if="$1" bs=1023k count=1 >> $GITHUB_STEP_SUMMARY } -# sanity -function summarize_s() { - headline="$1" +# generate summary of one test +function generate() { + VMs=3 + #################################################################### + # osname.txt -> used for headline + # disk-before.txt -> used together with uname + # disk-afterwards.txt -> used together with uname + # vm{1,2,3}log.txt (colored, used when current/log isn't there) + #################################################################### + # vm{1,2,3}/uname.txt -> used once + # vm{1,2,3}/build-stderr.txt -> used once + # vm{1,2,3}/dmesg-prerun.txt -> used once + # vm{1,2,3}/dmesg-module-load.txt -> used once + # vm{1,2,3}/console.txt -> all 3 used + #################################################################### + # vm{1,2,3}/current/log -> if not there, kernel panic loading + # vm{1,2,3}/current/results -> if not there, kernel panic testings + # vm{1,2,3}/exitcode.txt + #################################################################### + + # headline of this summary output "\n## $headline\n" - rm -rf testfiles - check_tarfile "$2/sanity.tar" - check_logfile "testfiles/log" - generate + + for i in `seq 1 $VMs`; do + if [ -s vm$i/uname.txt ]; then + output "
"
+      outfile vm$i/uname.txt
+      output "\nVM disk usage before:"
+      outfile disk-afterwards.txt
+      output "\nand afterwards:"
+      outfile disk-before.txt
+      output "
" + break + fi + done + + for i in `seq 1 $VMs`; do + if [ -s vm$i/build-stderr.txt ]; then + showfile "vm$i/build-stderr.txt" "Module build (stderr output)" + break + fi + done + + for i in `seq 1 $VMs`; do + if [ -s vm$i/dmesg-prerun.txt ]; then + showfile "vm$i/dmesg-prerun.txt" "Dmesg output - before tests" + break + fi + done + + for i in `seq 1 $VMs`; do + if [ -s vm$i/dmesg-module-load.txt ]; then + showfile "vm$i/dmesg-module-load.txt" "Dmesg output - module loading" + break + fi + done + + for i in `seq 1 $VMs`; do + log="vm$i/current/log" + if [ ! -f $log ]; then + output ":exclamation: Logfile of vm$i tests is missing :exclamation:" + + # some out may be generated + if [ -s vm${i}log.txt ]; then + cat vm${i}log.txt | \ + sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log" + showfile "vm${i}log" "Generated tests output of vm$i" + fi + + # output the console contents and continue with next vm + if [ -s "vm$i/console.txt" ]; then + cat "vm$i/console.txt" | \ + sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log" + showfile "vm${i}log" "Serial console output of vm$i" + fi + + rm -f "vm${i}log" + continue + fi + + cat $log | grep '^Test[: ]' > tests.txt + results="vm$i/current/results" + if [ ! -s "$results" ]; then + output ":exclamation: Results file of vm$i tests is missing :exclamation:" + # generate results file from log + ./zts-report.py --no-maybes ./tests.txt > $results + # Running Time: 01:30:09 + # Running Time: not finished!! + echo -e "\nRunning Time:\tKernel panic!" >> $results + fi + cat $results | awk '/Results Summary/ { show=1; print; next; } show' > summary.txt + runtime=`cat $results | grep '^Running Time:' | cut -f2` + + awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } \ + /\[SKIP\]|\[PASS\]/{ show=0; } show' $log > debug.txt + + output "\n### Tests on vm$i ($runtime)\n\n" + + if [ -s summary.txt ]; then + showfile "summary.txt" "Summary of all tests" + fi + + if [ -s "vm$i/console.txt" ]; then + showfile "vm$i/console.txt" "Serial console output" + fi + + if [ -s tests.txt ]; then + showfile "tests.txt" "List of all tests" + fi + + MAX="300" + if [ -s debug.txt ]; then + S=`stat --printf="%s" "debug.txt"` + if [ $S -gt $((1024*$MAX)) ]; then + dd if=debug.txt of=debug.txt2 count=$MAX bs=1024 2>/dev/null + mv -f debug.txt2 debug.txt + echo "..." >> debug.txt + echo "!!! THIS FILE IS BIGGER !!!" >> debug.txt + echo "Please download the zip archiv for full content!" >> debug.txt + fi + showfile "debug.txt" "Debug list for failed tests (vm$i, $runtime)" + fi + done } -# functional -function summarize_f() { - headline="$1" - output "\n## $headline\n" - rm -rf testfiles - for i in $(seq 1 $FUNCTIONAL_PARTS); do - tarfile="$2-part$i/part$i.tar" - check_tarfile "$tarfile" - check_logfile "testfiles/log" +# functional tests via qemu +function summarize() { + for tarfile in Logs-functional-*/qemu-*.tar; do + rm -rf vm* *.txt + tar xf "$tarfile" + osname=`cat osname.txt` + headline="Functional Tests: $osname" + generate done - generate } # https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits @@ -99,21 +178,18 @@ function summarize_f() { # [ ] can not show all error findings here # [x] split files into smaller ones and create additional steps -ERRLOGS=0 -if [ ! -f Summary/Summary.md ]; then - # first call, we do the default summary (~500k) - echo -n > Summary.md - summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity - summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity - summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional - summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional - - cat Summary.md >> $GITHUB_STEP_SUMMARY - mkdir -p Summary - mv *.md Summary +# first call, generate all summaries +if [ ! -f out-0.md ]; then + # create ./zts-report.py for generate() + TEMPLATE="tests/test-runner/bin/zts-report.py.in" + cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py + chmod +x ./zts-report.py + + logfile="0" + summarize + send2github out-0.md else - # here we get, when errors where returned in first call - test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY + send2github out-$1.md fi exit 0 diff --git a/.github/workflows/scripts/setup-dependencies.sh b/.github/workflows/scripts/setup-dependencies.sh deleted file mode 100755 index b40f9290f914..000000000000 --- a/.github/workflows/scripts/setup-dependencies.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -function prerun() { - echo "::group::Install build dependencies" - # remove snap things, update+upgrade will be faster then - for x in lxd core20 snapd; do sudo snap remove $x; done - sudo apt-get purge snapd google-chrome-stable firefox - # https://github.com/orgs/community/discussions/47863 - sudo apt-get remove grub-efi-amd64-bin grub-efi-amd64-signed shim-signed --allow-remove-essential - sudo apt-get update - sudo apt upgrade - sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq - sudo apt-get clean - sudo dmesg -c > /var/tmp/dmesg-prerun - echo "::endgroup::" -} - -function mod_build() { - echo "::group::Generate debian packages" - ./autogen.sh - ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan - make --no-print-directory --silent native-deb-utils native-deb-kmod - mv ../*.deb . - rm ./openzfs-zfs-dracut*.deb ./openzfs-zfs-dkms*.deb - echo "$ImageOS-$ImageVersion" > tests/ImageOS.txt - echo "::endgroup::" -} - -function mod_install() { - # install the pre-built module only on the same runner image - MOD=`cat tests/ImageOS.txt` - if [ "$MOD" != "$ImageOS-$ImageVersion" ]; then - rm -f *.deb - mod_build - fi - - echo "::group::Install and load modules" - # don't use kernel-shipped zfs modules - sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf - sudo apt-get install --fix-missing ./*.deb - - # Native Debian packages enable and start the services - # Stop zfs-zed daemon, as it may interfere with some ZTS test cases - sudo systemctl stop zfs-zed - sudo depmod -a - sudo modprobe zfs - sudo dmesg - sudo dmesg -c > /var/tmp/dmesg-module-load - echo "::endgroup::" - - echo "::group::Report CPU information" - lscpu - cat /proc/spl/kstat/zfs/chksum_bench - echo "::endgroup::" - - echo "::group::Optimize storage for ZFS testings" - # remove swap and umount fast storage - # 89GiB -> rootfs + bootfs with ~80MB/s -> don't care - # 64GiB -> /mnt with 420MB/s -> new testing ssd - sudo swapoff -a - - # this one is fast and mounted @ /mnt - # -> we reformat with ext4 + move it to /var/tmp - DEV="/dev/disk/azure/resource-part1" - sudo umount /mnt - sudo mkfs.ext4 -O ^has_journal -F $DEV - sudo mount -o noatime,barrier=0 $DEV /var/tmp - sudo chmod 1777 /var/tmp - - # disk usage afterwards - sudo df -h / - sudo df -h /var/tmp - sudo fstrim -a - echo "::endgroup::" -} - -case "$1" in - build) - prerun - mod_build - ;; - tests) - prerun - mod_install - ;; -esac diff --git a/.github/workflows/scripts/setup-functional.sh b/.github/workflows/scripts/setup-functional.sh deleted file mode 100755 index 08c4d872abdf..000000000000 --- a/.github/workflows/scripts/setup-functional.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -TDIR="/usr/share/zfs/zfs-tests/tests/functional" -echo -n "TODO=" -case "$1" in - part1) - # ~1h 20m - echo "cli_root" - ;; - part2) - # ~1h - ls $TDIR|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ',' - ;; - part3) - # ~1h - ls $TDIR|grep '^[n-qs-z]'|xargs|tr -s ' ' ',' - ;; - part4) - # ~1h - ls $TDIR|grep '^r'|xargs|tr -s ' ' ',' - ;; -esac diff --git a/.github/workflows/zfs-linux-tests.yml b/.github/workflows/zfs-linux-tests.yml deleted file mode 100644 index 753f3cd0214e..000000000000 --- a/.github/workflows/zfs-linux-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: zfs-linux-tests - -on: - workflow_call: - inputs: - os: - description: 'The ubuntu version: 20.02 or 22.04' - required: true - type: string - -jobs: - - zloop: - runs-on: ubuntu-${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v4 - with: - name: modules-${{ inputs.os }} - - name: Install modules - run: | - tar xzf modules-${{ inputs.os }}.tgz - .github/workflows/scripts/setup-dependencies.sh tests - - name: Tests - timeout-minutes: 30 - run: | - sudo mkdir -p /var/tmp/zloop - # run for 10 minutes or at most 2 iterations for a maximum runner - # time of 20 minutes. - sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60 - - name: Prepare artifacts - if: failure() - run: | - sudo chmod +r -R /var/tmp/zloop/ - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: Zpool-logs-${{ inputs.os }} - path: | - /var/tmp/zloop/*/ - !/var/tmp/zloop/*/vdev/ - retention-days: 14 - if-no-files-found: ignore - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: Zpool-files-${{ inputs.os }} - path: | - /var/tmp/zloop/*/vdev/ - retention-days: 14 - if-no-files-found: ignore - - sanity: - runs-on: ubuntu-${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v4 - with: - name: modules-${{ inputs.os }} - - name: Install modules - run: | - tar xzf modules-${{ inputs.os }}.tgz - .github/workflows/scripts/setup-dependencies.sh tests - - name: Tests - timeout-minutes: 60 - shell: bash - run: | - set -o pipefail - /usr/share/zfs/zfs-tests.sh -vKR -s 3G -r sanity | scripts/zfs-tests-color.sh - - name: Prepare artifacts - if: success() || failure() - run: | - RESPATH="/var/tmp/test_results" - mv -f $RESPATH/current $RESPATH/testfiles - tar cf $RESPATH/sanity.tar -h -C $RESPATH testfiles - - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: Logs-${{ inputs.os }}-sanity - path: /var/tmp/test_results/sanity.tar - if-no-files-found: ignore - - functional: - runs-on: ubuntu-${{ inputs.os }} - strategy: - fail-fast: false - matrix: - tests: [ part1, part2, part3, part4 ] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v4 - with: - name: modules-${{ inputs.os }} - - name: Install modules - run: | - tar xzf modules-${{ inputs.os }}.tgz - .github/workflows/scripts/setup-dependencies.sh tests - - name: Setup tests - run: | - .github/workflows/scripts/setup-functional.sh ${{ matrix.tests }} >> $GITHUB_ENV - - name: Tests - timeout-minutes: 120 - shell: bash - run: | - set -o pipefail - /usr/share/zfs/zfs-tests.sh -vKR -s 3G -T ${{ env.TODO }} | scripts/zfs-tests-color.sh - - name: Prepare artifacts - if: success() || failure() - run: | - RESPATH="/var/tmp/test_results" - mv -f $RESPATH/current $RESPATH/testfiles - tar cf $RESPATH/${{ matrix.tests }}.tar -h -C $RESPATH testfiles - - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: Logs-${{ inputs.os }}-functional-${{ matrix.tests }} - path: /var/tmp/test_results/${{ matrix.tests }}.tar - if-no-files-found: ignore diff --git a/.github/workflows/zfs-linux.yml b/.github/workflows/zfs-linux.yml deleted file mode 100644 index e6b705c86055..000000000000 --- a/.github/workflows/zfs-linux.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: zfs-linux - -on: - push: - pull_request: - -jobs: - - build: - name: Build - strategy: - fail-fast: false - matrix: - os: [20.04, 22.04] - runs-on: ubuntu-${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Build modules - run: .github/workflows/scripts/setup-dependencies.sh build - - name: Prepare modules upload - run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt - - uses: actions/upload-artifact@v4 - with: - name: modules-${{ matrix.os }} - path: modules-${{ matrix.os }}.tgz - retention-days: 14 - - testings: - name: Testing - strategy: - fail-fast: false - matrix: - os: [20.04, 22.04] - needs: build - uses: ./.github/workflows/zfs-linux-tests.yml - with: - os: ${{ matrix.os }} - - cleanup: - if: always() - name: Cleanup - runs-on: ubuntu-22.04 - needs: testings - steps: - - uses: actions/download-artifact@v4 - - name: Generating summary - run: | - tar xzf modules-22.04/modules-22.04.tgz .github tests - .github/workflows/scripts/generate-summary.sh - # up to 4 steps, each can have 1 MiB output (for debugging log files) - - name: Summary for errors #1 - run: .github/workflows/scripts/generate-summary.sh 1 - - name: Summary for errors #2 - run: .github/workflows/scripts/generate-summary.sh 2 - - name: Summary for errors #3 - run: .github/workflows/scripts/generate-summary.sh 3 - - name: Summary for errors #4 - run: .github/workflows/scripts/generate-summary.sh 4 - - uses: actions/upload-artifact@v4 - with: - name: Summary Files - path: Summary/ diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh index 85854085f560..b5d8a4fc32ae 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh @@ -67,6 +67,7 @@ log_onexit cleanup cleanup + typeset -i i=0 while (( i < ${#shareopts[*]} )) do diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh index ec11610742fb..e7588e207b46 100755 --- a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh @@ -68,6 +68,7 @@ log_must mount_redacted -f $recvfs # when we re-enable redaction blkptrs. # #log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1 +log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt" log_must diff $send_mnt/f2 $recv_mnt/f2 log_must zfs rollback -R $clone@snap log_must zfs destroy -R $recvfs @@ -96,6 +97,7 @@ log_must zfs destroy -R $clone2 log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream" log_must eval "zfs recv $recvfs <$stream" log_must mount_redacted -f $recvfs +log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt" log_must [ "$(ls $send_mnt)" == "$(ls $recv_mnt)" ] log_must zfs destroy -R $recvfs log_must zfs rollback -R $sendfs@snap