diff --git a/.github/workflows/scripts/README.md b/.github/workflows/scripts/README.md new file mode 100644 index 000000000000..1f2f76499f05 --- /dev/null +++ b/.github/workflows/scripts/README.md @@ -0,0 +1,10 @@ + +Workflow for each operating system: + - install QEMU on the github runner + - download current cloud image + - start and init that image via cloud-init + - install deps and poweroff system + - start system and build openzfs and then poweroff again + - clone the system and start 4 qemu workers for the testings (4x 3GB RAM) + - use trimable virtual disks (3x 1GB) for each testing system + - do the functional testings < 3h for each os diff --git a/.github/workflows/scripts/generate-summary.sh b/.github/workflows/scripts/generate-summary.sh index b5d89208a5d8..d5e6d91cc30b 100755 --- a/.github/workflows/scripts/generate-summary.sh +++ b/.github/workflows/scripts/generate-summary.sh @@ -1,97 +1,176 @@ #!/usr/bin/env bash -# for runtime reasons we split functional testings into N parts -# - use a define to check for missing tarfiles -FUNCTIONAL_PARTS="4" - -ZTS_REPORT="tests/test-runner/bin/zts-report.py" -chmod +x $ZTS_REPORT +###################################################################### +# generate github summary page of all the testings +# /tr 2024-06-17 +###################################################################### function output() { - echo -e $* >> Summary.md -} - -function error() { - output ":bangbang: $* :bangbang:\n" + echo -e $* >> "out-$logfile.md" } -# this function generates the real summary -# - expects a logfile "log" in current directory -function generate() { - # we issued some error already - test ! -s log && return - - # for overview and zts-report - cat log | grep '^Test' > list - - # error details - awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } - /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err - - # summary of errors - if [ -s err ]; then - output "
"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
-    output "
" - - # generate seperate error logfile - ERRLOGS=$((ERRLOGS+1)) - errfile="err-$ERRLOGS.md" - echo -e "\n## $headline (debugging)\n" >> $errfile - echo "
Error Listing - with dmesg and dbgmsg
" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "
" >> $errfile - else - output "All tests passed :thumbsup:" +function outfile() { + if [ -f $1 ]; then + CUR=`stat --printf="%s" "out-$logfile.md"` + ADD=`stat --printf="%s" "$1"` + X=$((CUR+ADD)) + if [ $X -lt $((1024*1023)) ]; then + cat "$1" >> "out-$logfile.md" + else + logfile=$((logfile+1)) + cat "$1" >> "out-$logfile.md" + fi fi - - output "
Full Listing
"
-  cat list >> Summary.md
-  output "
" - - # remove tmp files - rm -f err list log } -# check tarfiles and untar -function check_tarfile() { - if [ -f "$1" ]; then - tar xf "$1" || error "Tarfile $1 returns some error" - else - error "Tarfile $1 not found" - fi +function showfile() { + filename="$1" + headline="$2" + echo "
$headline
" > tmp
+  cat $filename >> tmp
+  echo "
" >> tmp + outfile tmp + rm -f tmp } -# check logfile and concatenate test results -function check_logfile() { - if [ -f "$1" ]; then - cat "$1" >> log - else - error "Logfile $1 not found" - fi +function send2github() { + test -f "$1" && dd if="$1" bs=1023k count=1 >> $GITHUB_STEP_SUMMARY } -# sanity -function summarize_s() { - headline="$1" +# generate summary of one test +function generate() { + VMs=3 + #################################################################### + # osname.txt -> used for headline + # disk-before.txt -> used together with uname + # disk-afterwards.txt -> used together with uname + # vm{1,2,3}log.txt (colored, used when current/log isn't there) + #################################################################### + # vm{1,2,3}/uname.txt -> used once + # vm{1,2,3}/build-stderr.txt -> used once + # vm{1,2,3}/dmesg-prerun.txt -> used once + # vm{1,2,3}/dmesg-module-load.txt -> used once + # vm{1,2,3}/console.txt -> all 3 used + #################################################################### + # vm{1,2,3}/current/log -> if not there, kernel panic loading + # vm{1,2,3}/current/results -> if not there, kernel panic testings + # vm{1,2,3}/exitcode.txt + #################################################################### + + # headline of this summary output "\n## $headline\n" - rm -rf testfiles - check_tarfile "$2/sanity.tar" - check_logfile "testfiles/log" - generate + + for i in `seq 1 $VMs`; do + if [ -s vm$i/uname.txt ]; then + output "
"
+      outfile vm$i/uname.txt
+      output "\nVM disk usage before:"
+      outfile disk-afterwards.txt
+      output "\nand afterwards:"
+      outfile disk-before.txt
+      output "
" + break + fi + done + + for i in `seq 1 $VMs`; do + if [ -s vm$i/build-stderr.txt ]; then + showfile "vm$i/build-stderr.txt" "Module build (stderr output)" + break + fi + done + + for i in `seq 1 $VMs`; do + if [ -s vm$i/dmesg-prerun.txt ]; then + showfile "vm$i/dmesg-prerun.txt" "Dmesg output - before tests" + break + fi + done + + for i in `seq 1 $VMs`; do + if [ -s vm$i/dmesg-module-load.txt ]; then + showfile "vm$i/dmesg-module-load.txt" "Dmesg output - module loading" + break + fi + done + + for i in `seq 1 $VMs`; do + log="vm$i/current/log" + if [ ! -f $log ]; then + output ":exclamation: Logfile of vm$i tests is missing :exclamation:" + + # some out may be generated + if [ -s vm${i}log.txt ]; then + cat vm${i}log.txt | \ + sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log" + showfile "vm${i}log" "Generated tests output of vm$i" + fi + + # output the console contents and continue with next vm + if [ -s "vm$i/console.txt" ]; then + cat "vm$i/console.txt" | \ + sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log" + showfile "vm${i}log" "Serial console output of vm$i" + fi + + rm -f "vm${i}log" + continue + fi + + cat $log | grep '^Test[: ]' > tests.txt + results="vm$i/current/results" + if [ ! -s "$results" ]; then + output ":exclamation: Results file of vm$i tests is missing :exclamation:" + # generate results file from log + ./zts-report.py --no-maybes ./tests.txt > $results + # Running Time: 01:30:09 + # Running Time: not finished!! + echo -e "\nRunning Time:\tKernel panic!" >> $results + fi + cat $results | awk '/Results Summary/ { show=1; print; next; } show' > summary.txt + runtime=`cat $results | grep '^Running Time:' | cut -f2` + + awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } \ + /\[SKIP\]|\[PASS\]/{ show=0; } show' $log > debug.txt + + output "\n### Tests on vm$i ($runtime)\n\n" + + if [ -s summary.txt ]; then + showfile "summary.txt" "Summary of all tests" + fi + + if [ -s "vm$i/console.txt" ]; then + showfile "vm$i/console.txt" "Serial console output" + fi + + if [ -s tests.txt ]; then + showfile "tests.txt" "List of all tests" + fi + + MAX="300" + if [ -s debug.txt ]; then + S=`stat --printf="%s" "debug.txt"` + if [ $S -gt $((1024*$MAX)) ]; then + dd if=debug.txt of=debug.txt2 count=$MAX bs=1024 2>/dev/null + mv -f debug.txt2 debug.txt + echo "..." >> debug.txt + echo "!!! THIS FILE IS BIGGER !!!" >> debug.txt + echo "Please download the zip archiv for full content!" >> debug.txt + fi + showfile "debug.txt" "Debug list for failed tests (vm$i, $runtime)" + fi + done } -# functional -function summarize_f() { - headline="$1" - output "\n## $headline\n" - rm -rf testfiles - for i in $(seq 1 $FUNCTIONAL_PARTS); do - tarfile="$2-part$i/part$i.tar" - check_tarfile "$tarfile" - check_logfile "testfiles/log" +# functional tests via qemu +function summarize() { + for tarfile in Logs-functional-*/qemu-*.tar; do + rm -rf vm* *.txt + tar xf "$tarfile" + osname=`cat osname.txt` + headline="Functional Tests: $osname" + generate done - generate } # https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits @@ -99,21 +178,18 @@ function summarize_f() { # [ ] can not show all error findings here # [x] split files into smaller ones and create additional steps -ERRLOGS=0 -if [ ! -f Summary/Summary.md ]; then - # first call, we do the default summary (~500k) - echo -n > Summary.md - summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity - summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity - summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional - summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional - - cat Summary.md >> $GITHUB_STEP_SUMMARY - mkdir -p Summary - mv *.md Summary +# first call, generate all summaries +if [ ! -f out-0.md ]; then + # create ./zts-report.py for generate() + TEMPLATE="tests/test-runner/bin/zts-report.py.in" + cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py + chmod +x ./zts-report.py + + logfile="0" + summarize + send2github out-0.md else - # here we get, when errors where returned in first call - test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY + send2github out-$1.md fi exit 0 diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh new file mode 100755 index 000000000000..08c9b37e2efd --- /dev/null +++ b/.github/workflows/scripts/qemu-1-setup.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +###################################################################### +# 1) setup the action runner to start some qemu instance +###################################################################### + +set -eu + +# docker isn't needed, free some memory +sudo systemd-run --wait docker system prune --force --all --volumes +sudo systemctl stop docker.socket +sudo apt-get remove docker-ce-cli docker-ce podman + +# remove unneeded things +sudo apt-get remove google-chrome-stable snapd + +# install needed packages +sudo apt-get update +sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \ + virt-manager linux-modules-extra-`uname -r` + +# remove unused software +df -h / +sudo systemd-run --wait rm -rf \ + /opt/* \ + /usr/local/* \ + /usr/share/az* \ + /usr/share/dotnet \ + /usr/share/gradle* \ + /usr/share/miniconda \ + /usr/share/swift \ + /var/lib/gems \ + /var/lib/mysql \ + /var/lib/snapd + +# disk usage afterwards +sudo df -h / +sudo df -h /mnt +sudo fstrim -a + +# generate ssh keys +ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N "" diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh new file mode 100755 index 000000000000..cf0198e013eb --- /dev/null +++ b/.github/workflows/scripts/qemu-2-start.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +###################################################################### +# 2) start qemu with some operating system, init via cloud-init +###################################################################### + +set -eu + +# valid ostypes: virt-install --os-variant list +OS="$1" +OSv=$OS + +# compressed with .zst extension +URLzs="" + +case "$OS" in + almalinux8) + OSNAME="AlmaLinux 8" + URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2" + ;; + almalinux9) + OSNAME="AlmaLinux 9" + URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2" + ;; + archlinux) + OSNAME="Archlinux" + URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2" + ;; + centos-stream9) + OSNAME="CentOS Stream 9" + URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" + ;; + debian11) + OSNAME="Debian 11" + URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2" + ;; + debian12) + OSNAME="Debian 12" + URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2" + ;; + fedora39) + OSNAME="Fedora 39" + OSv="fedora39" + URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2" + ;; + fedora40) + OSNAME="Fedora 40" + OSv="fedora39" + URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2" + ;; + freebsd13) + OSNAME="FreeBSD 13" + OSv="freebsd13.0" + # URL="https://download.freebsd.org/ftp/snapshots/amd64" + # freebsd images don't have clout-init within it! :( + # -> workaround: provide own images + URLzs="https://openzfs.de/freebsd/amd64-freebsd-13.3.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + freebsd14) + OSNAME="FreeBSD 14" + OSv="freebsd14.0" + URLzs="https://openzfs.de/freebsd/amd64-freebsd-14.0.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + freebsd15) + OSNAME="FreeBSD 15" + OSv="freebsd14.0" + URLzs="https://openzfs.de/freebsd/amd64-freebsd-15.0.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + ubuntu22) + OSNAME="Ubuntu 22.04" + OSv="ubuntu22.04" + #URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" + #URL="https://mirrors.cloud.tencent.com/ubuntu-cloud-images/jammy/current/jammy-server-cloudimg-amd64.img" + URL="https://mirror.citrahost.com/ubuntu-cloud-images/jammy/current/jammy-server-cloudimg-amd64.img" + ;; + ubuntu24) + OSNAME="Ubuntu 24.04" + OSv="ubuntu24.04" + #URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + #URL="https://mirrors.cloud.tencent.com/ubuntu-cloud-images/noble/current/noble-server-cloudimg-amd64.img" + URL="https://mirror.citrahost.com/ubuntu-cloud-images/noble/current/noble-server-cloudimg-amd64.img" + ;; + *) + echo "Wrong value for variable OS!" + exit 111 + ;; +esac + +IMG="/mnt/cloudimg.qcow2" +DISK="/mnt/openzfs.qcow2" +sudo chown -R $(whoami) /mnt + +if [ ! -z "$URLzs" ]; then + echo "Loading image $URLzs ..." + time axel -q -o "$IMG.zst" "$URLzs" || exit 111 + zstd -q -d --rm "$IMG.zst" +else + echo "Loading image $URL ..." + time axel -q -o "$IMG" "$URL" || exit 111 +fi + +# for later use +echo "$OS" > /var/tmp/os.txt +echo "$OSv" > /var/tmp/osvariant.txt +echo "$OSNAME" > /var/tmp/osname.txt + +# we use zstd for faster IO on the testing runner +echo "Converting image ..." +qemu-img convert -q -f qcow2 -O qcow2 -c \ + -o compression_type=zstd,preallocation=off $IMG $DISK || exit 111 +rm -f $IMG || exit 111 + +echo "Resizing image to 20GiB ..." +qemu-img resize -q $DISK 20G || exit 111 + +PUBKEY=`cat ~/.ssh/id_ed25519.pub` +cat < /tmp/user-data +#cloud-config + +fqdn: $OS + +# user:zfs password:1 +users: +- name: root + shell: $BASH +- name: zfs + sudo: ALL=(ALL) NOPASSWD:ALL + shell: $BASH + lock-passwd: false + passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/ + ssh_authorized_keys: + - $PUBKEY + +growpart: + mode: auto + devices: ['/'] + ignore_growroot_disabled: false +EOF + +for i in `seq 0 3`; do + sudo virsh net-update default add ip-dhcp-host \ + "" --live --config +done + +sudo virt-install \ + --os-variant $OSv \ + --name "openzfs" \ + --cpu host-passthrough \ + --virt-type=kvm --hvm \ + --vcpus=4,sockets=1 \ + --memory $((1024*4)) \ + --memballoon model=none \ + --graphics none \ + --network bridge=virbr0,model=virtio,mac='52:54:00:83:79:00' \ + --cloud-init user-data=/tmp/user-data \ + --disk $DISK,format=qcow2,bus=virtio \ + --import --noautoconsole 2>/dev/null diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh new file mode 100755 index 000000000000..ec06d32e3610 --- /dev/null +++ b/.github/workflows/scripts/qemu-3-deps.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +###################################################################### +# 3) install dependencies for compiling and loading +###################################################################### + +set -eu + +function archlinux() { + echo "##[group]Running pacman -Syu" + sudo pacman -Syu --noconfirm + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \ + fio gdb inetutils less linux linux-headers lsscsi nfs-utils parted pax \ + perf python-packaging python-setuptools ksh samba sysstat rng-tools \ + rsync wget + echo "##[endgroup]" +} + +function debian() { + export DEBIAN_FRONTEND="noninteractive" + + echo "##[group]Running apt-get update+upgrade" + sudo apt-get update -y + sudo apt-get upgrade -y + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo apt-get install -y \ + acl alien attr autoconf bc cpio curl dbench dh-python \ + dh-sequence-dkms dkms fakeroot fio gdb gdebi git ksh lcov \ + isc-dhcp-client libacl1-dev libaio-dev libattr1-dev libblkid-dev \ + libcurl4-openssl-dev libdevmapper-dev libelf-dev libffi-dev \ + libmount-dev libpam0g-dev libselinux-dev libssl-dev libtool \ + libtool-bin libudev-dev linux-headers-$(uname -r) lsscsi \ + nfs-kernel-server pamtester parted python3 python3-all-dev \ + python3-cffi python3-dev python3-distlib python3-packaging \ + python3-setuptools python3-sphinx rng-tools rpm2cpio rsync samba \ + sysstat uuid-dev watchdog wget xfslibs-dev zlib1g-dev + echo "##[endgroup]" +} + +function freebsd() { + export ASSUME_ALWAYS_YES="YES" + + echo "##[group]Install Development Tools" + sudo pkg install -y autoconf automake autotools base64 fio gdb git \ + gmake gsed python python3 gettext gettext-runtime checkbashisms lcov \ + libtool lscpu ksh93 pamtester pamtester rsync + sudo pkg install -xy \ + '^samba4[[:digit:]]+$' \ + '^py3[[:digit:]]+-cffi$' \ + '^py3[[:digit:]]+-sysctl$' \ + '^py3[[:digit:]]+-packaging$' + echo "##[endgroup]" +} + +# common packages for: almalinux, centos, redhat +function rhel() { + echo "##[group]Running dnf update" + sudo dnf update -y + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo dnf group install -y "Development Tools" + sudo dnf install -y \ + acl attr bc bzip2 curl dbench dkms elfutils-libelf-devel fio gdb git \ + kernel-rpm-macros ksh libacl-devel libaio-devel libargon2-devel \ + libattr-devel libblkid-devel libcurl-devel libffi-devel ncompress \ + libselinux-devel libtirpc-devel libtool libudev-devel libuuid-devel \ + lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \ + python3 python3-cffi python3-devel python3-packaging kernel-devel \ + python3-setuptools rng-tools rpcgen rpm-build rsync samba sysstat \ + systemd watchdog wget xfsprogs-devel zlib-devel + echo "##[endgroup]" +} + +# Install dependencies +case "$1" in + almalinux8) + echo "##[group]Enable epel and powertools repositories" + sudo dnf config-manager -y --set-enabled powertools + sudo dnf install -y epel-release + echo "##[endgroup]" + rhel + echo "##[group]Install kernel-abi-whitelists" + sudo dnf install -y kernel-abi-whitelists + echo "##[endgroup]" + ;; + almalinux9|centos-stream9) + echo "##[group]Enable epel and crb repositories" + sudo dnf config-manager -y --set-enabled crb + sudo dnf install -y epel-release + echo "##[endgroup]" + rhel + echo "##[group]Install kernel-abi-stablelists" + sudo dnf install -y kernel-abi-stablelists + echo "##[endgroup]" + ;; + archlinux) + archlinux + ;; + debian*) + debian + echo "##[group]Install linux-perf" + sudo apt-get install -yq linux-perf + echo "##[endgroup]" + ;; + fedora*) + rhel + ;; + freebsd*) + freebsd + ;; + ubuntu*) + debian + echo "##[group]Install linux-tools-common" + sudo apt-get install -yq linux-tools-common libtirpc-dev + echo "##[endgroup]" + ;; +esac + +# Start services +echo "##[group]Enable services" +case "$1" in + freebsd*) + echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab + sudo -E mount /dev/fd + sudo -E touch /etc/zfs/exports + sudo -E sysrc mountd_flags="/etc/zfs/exports" + sudo -E service nfsd enable + echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null + sudo -E service samba_server enable + ;; + debian*|ubuntu*) + sudo -E systemctl enable nfs-kernel-server + sudo -E systemctl enable smbd + ;; + *) + # All other linux distros + sudo -E systemctl enable nfs-server + sudo -E systemctl enable smb + ;; +esac +echo "##[endgroup]" + +# Enable serial console and remove 'quiet' from linux kernel cmdline +case "$1" in + freebsd*) + true + ;; + *) + echo "##[group]Enable serial output" + sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0,115200n8 random.trust_cpu=on/g; s/quiet //g' /etc/default/grub || true + for i in /boot/grub/grub.cfg /etc/grub2.cfg /etc/grub2-efi.cfg /boot/grub2/grub.cfg ; do + test -e $i || continue + echo sudo grub-mkconfig -o $i + sudo grub-mkconfig -o $i + done + echo "##[endgroup]" + ;; +esac + +# ssh config +mkdir -p $HOME/.ssh +echo "StrictHostKeyChecking no" >> $HOME/.ssh/config +echo "ConnectTimeout 1" >> $HOME/.ssh/config + +# poweroff the machine +# - use it for building openzfs when restarted +sleep 2 && sudo poweroff & +exit 0 diff --git a/.github/workflows/scripts/qemu-4-build.sh b/.github/workflows/scripts/qemu-4-build.sh new file mode 100755 index 000000000000..4f2b9d2d7335 --- /dev/null +++ b/.github/workflows/scripts/qemu-4-build.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +###################################################################### +# 4) configure and build openzfs modules +###################################################################### + +set -eu + +# FD 3 used for stdout +exec 3>&1 + +function run() { + LOG="/var/tmp/build-stderr.txt" + echo "**************************************************" >> $LOG + echo "`date` ($*)" + echo "**************************************************" >> $LOG + $@ 2>&1 1>&3 | tee -a $LOG +} + +function freebsd() { + export MAKE="gmake" + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr/local \ + --with-libintl-prefix=/usr/local \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run gmake -j`sysctl -n hw.ncpu` + echo "##[endgroup]" + + echo "##[group]Install" + run sudo gmake install + echo "##[endgroup]" +} + +function linux() { + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run make -j$(nproc) + echo "##[endgroup]" + + echo "##[group]Install" + run sudo make install + echo "##[endgroup]" +} + +function rpm_build_and_install() { + EXTRA_CONFIG="${1:-}" + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG + echo "##[endgroup]" + + echo "##[group]Build" + run make pkg-kmod pkg-utils + echo "##[endgroup]" + + echo "##[group]Install" + run sudo yum -y --skip-broken localinstall $(ls *.rpm | grep -v src.rpm) + echo "##[endgroup]" + +} + +function deb_build_and_install() { +echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run make native-deb-kmod native-deb-utils + echo "##[endgroup]" + + echo "##[group]Install" + # Do kmod install. Note that when you build the native debs, the + # packages themselves are placed in parent directory '../' rather than + # in the source directory like the rpms are. + run sudo apt-get -y install `find ../ | grep -E '\.deb$' | grep -Ev 'dkms|dracut'` + echo "##[endgroup]" +} + +# Debug: show kernel cmdline +if [ -e /proc/cmdline ] ; then + cat /proc/cmdline || true +fi + +cd $HOME/zfs +export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" + +# remove old configuration files +sudo rm -rf /var/lib/cloud \ +/etc/systemd/network/10-cloud-init-eth0.network \ +/etc/network/interfaces.d/50-cloud-init.cfg \ +/etc/netplan/50-cloud-init.yaml + +# build +case "$1" in + freebsd*) + freebsd + ;; + alma*|centos*) + rpm_build_and_install "--with-spec=redhat" + ;; + fedora*) + rpm_build_and_install + ;; + debian*|ubuntu*) + deb_build_and_install + ;; + *) + linux + ;; +esac + +# poweroff the machine +# - use it for the testings when restarted +sync +sleep 2 && sudo poweroff & +exit 0 diff --git a/.github/workflows/scripts/qemu-5-setup.sh b/.github/workflows/scripts/qemu-5-setup.sh new file mode 100755 index 000000000000..22be2555e7a6 --- /dev/null +++ b/.github/workflows/scripts/qemu-5-setup.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +###################################################################### +# 5) start test machines and load openzfs module +###################################################################### + +set -eu + +# machine not needed anymore +while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do sleep 1; done +sudo virsh undefine openzfs + +############################################################### +# 1) 4GB RAM for host +# 2) 3x 4GB RAM for qemu +############################################################### +PUBKEY=`cat ~/.ssh/id_ed25519.pub` +OSv=`cat /var/tmp/osvariant.txt` +OS=`cat /var/tmp/os.txt` +for i in `seq 1 3`; do + + if [ $i = 1 ]; then + # 8 vdisks on /mnt + BASE="/mnt" + else + # 4 vdisks on / + BASE="/mnt2" + fi + + OPTS="-q -f qcow2 -o compression_type=zstd,preallocation=off" + SIZE="2" + + echo "Generating vm$i with 3x${SIZE}GiB disks." + sudo mkdir -p $BASE + sudo qemu-img create $OPTS -b /mnt/openzfs.qcow2 -F qcow2 "$BASE/vm$i.qcow2" + sudo qemu-img create $OPTS "$BASE/vm$i-1.qcow2" ${SIZE}G + sudo qemu-img create $OPTS "$BASE/vm$i-2.qcow2" ${SIZE}G + sudo qemu-img create $OPTS "$BASE/vm$i-3.qcow2" ${SIZE}G + + cat < /tmp/user-data +#cloud-config + +fqdn: vm$i + +# user:zfs password:1 +users: +- name: root + shell: $BASH +- name: zfs + sudo: ALL=(ALL) NOPASSWD:ALL + shell: $BASH + lock-passwd: false + passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/ + ssh_authorized_keys: + - $PUBKEY + +growpart: + mode: auto + devices: ['/'] + ignore_growroot_disabled: false +EOF + + sudo virt-install \ + --os-variant $OSv \ + --name "vm$i" \ + --cpu host-passthrough \ + --virt-type=kvm --hvm \ + --vcpus=3,sockets=1 \ + --memory $((1024*4)) \ + --memballoon model=none \ + --graphics none \ + --cloud-init user-data=/tmp/user-data \ + --network bridge=virbr0,model=virtio,mac="52:54:00:83:79:0$i" \ + --disk "$BASE/vm$i.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \ + --disk "$BASE/vm$i-1.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \ + --disk "$BASE/vm$i-2.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \ + --disk "$BASE/vm$i-3.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \ + --import --noautoconsole >/dev/null +done + +# check if the machines are okay +echo "Waiting for vm's to come up..." +while true; do ssh 2>/dev/null zfs@192.168.122.11 "uname -a" && break; done +while true; do ssh 2>/dev/null zfs@192.168.122.12 "uname -a" && break; done +while true; do ssh 2>/dev/null zfs@192.168.122.13 "uname -a" && break; done diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh new file mode 100755 index 000000000000..2840c58aeac1 --- /dev/null +++ b/.github/workflows/scripts/qemu-6-tests.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +###################################################################### +# 6) load openzfs module and run the tests +###################################################################### + +set -o pipefail + +if [ -z "$1" ]; then + # called directly on the runner + P="/var/tmp" + + cd $P + OS=`cat os.txt` + IP1="192.168.122.11" + IP2="192.168.122.12" + IP3="192.168.122.13" + + df -h / /mnt > /var/tmp/disk-before.txt + + # start as daemon and log stdout + SSH=`which ssh` + CMD='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh' + daemonize -c $P -p vm1.pid -o vm1log.txt -- \ + $SSH zfs@$IP1 $CMD $OS part1 + daemonize -c $P -p vm2.pid -o vm2log.txt -- \ + $SSH zfs@$IP2 $CMD $OS part2 + daemonize -c $P -p vm3.pid -o vm3log.txt -- \ + $SSH zfs@$IP3 $CMD $OS part3 + + # give us the output of stdout + stderr - with prefix ;) + tail -fq vm1log.txt | sed -e "s/^/vm1: /g" & + tail -fq vm2log.txt | sed -e "s/^/vm2: /g" & + tail -fq vm3log.txt | sed -e "s/^/vm3: /g" & + + # wait for all vm's to finnish + tail --pid=`cat vm1.pid` -f /dev/null + tail --pid=`cat vm2.pid` -f /dev/null + tail --pid=`cat vm3.pid` -f /dev/null + + # kill the tail/sed combo + killall tail + df -h / /mnt > /var/tmp/disk-afterwards.txt + exit 0 +fi + +function freebsd() { + # when freebsd zfs is loaded, unload this one + kldstat -n zfs 2>/dev/null && sudo kldunload zfs + sudo dmesg -c > /var/tmp/dmesg-prerun.txt + sudo -E ./scripts/zfs.sh + sudo dmesg -c > /var/tmp/dmesg-module-load.txt + sudo kldstat -n openzfs +} + +function linux() { + sudo dmesg -c > /var/tmp/dmesg-prerun.txt + sudo -E modprobe zfs + sudo dmesg -c > /var/tmp/dmesg-module-load.txt +} + +function gettests() { + TF="$TDIR/zfs-tests/tests/functional" + echo -n "-T " + case "$1" in + part1) + # ~1h 40m (archlinux) + echo "cli_root" + ;; + part2) + # ~2h 5m (archlinux) + ls $TF|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ',' + ;; + part3) + # ~2h + ls $TF|grep '^[n-z]'|xargs|tr -s ' ' ',' + ;; + esac +} + +function gettestsD() { + TF="$TDIR/zfs-tests/tests/functional" + echo -n "-T " + case "$1" in + part1) + echo "checksum" + ;; + part2) + echo "casenorm,trim" + ;; + part3) + echo "pool_checkpoint" + ;; + esac +} + +# called within vm +export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin" + +case "$1" in + freebsd*) + TDIR="/usr/local/share/zfs" + freebsd + OPTS=`gettests $2` + if [ -e /dev/vtbd1 ] && [ -e /dev/vtbd2 ] && [ -e /dev/vtbd3 ] ; then + DISKS="/dev/vtbd1 /dev/vtbd2 /dev/vtbd3" + export DISKS + fi + ;; + *) + TDIR="/usr/share/zfs" + OPTS=`gettests $2` + linux + if [ -e /dev/vdb ] && [ -e /dev/vdc ] && [ -e /dev/vdd ] ; then + DISKS="/dev/vdb /dev/vdc /dev/vdd" + export DISKS + fi + ;; +esac + +# this part runs inside qemu, finally: run tests +uname -a > /var/tmp/uname.txt +cd $HOME/zfs +$TDIR/zfs-tests.sh -vKR -s 3G $OPTS | scripts/zfs-tests-color.sh +RV=$? +echo $RV > /var/tmp/exitcode.txt +exit $RV diff --git a/.github/workflows/scripts/qemu-7-reports.sh b/.github/workflows/scripts/qemu-7-reports.sh new file mode 100755 index 000000000000..924b783d937d --- /dev/null +++ b/.github/workflows/scripts/qemu-7-reports.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +###################################################################### +# 7) output the results of the previous stage in an ordered way +###################################################################### + +set -o pipefail +cd /var/tmp + +echo "VM disk usage before:" +cat disk-before.txt +echo "and afterwards:" +cat disk-afterwards.txt + +exitcode=0 +for i in `seq 1 3`; do + f="exitcode.vm$i" + scp 2>/dev/null zfs@192.168.122.1$i:/var/tmp/exitcode.txt $f + test -f $f || echo 2 > $f + + rv=`cat $f` + if [ $rv != 0 ]; then + msg="FAIL -> rv=$rv" + exitcode=$rv + else + msg="PASS" + fi + + echo "##[group]Summary vm$i [$msg]" + cat "vm${i}log.txt" | grep -v 'Test[ :]' + echo "##[endgroup]" + + echo "##[group]Results vm$i [$msg]" + cat "vm${i}log.txt" + echo "##[endgroup]" +done + +exit $exitcode diff --git a/.github/workflows/scripts/setup-dependencies.sh b/.github/workflows/scripts/setup-dependencies.sh deleted file mode 100755 index b40f9290f914..000000000000 --- a/.github/workflows/scripts/setup-dependencies.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -function prerun() { - echo "::group::Install build dependencies" - # remove snap things, update+upgrade will be faster then - for x in lxd core20 snapd; do sudo snap remove $x; done - sudo apt-get purge snapd google-chrome-stable firefox - # https://github.com/orgs/community/discussions/47863 - sudo apt-get remove grub-efi-amd64-bin grub-efi-amd64-signed shim-signed --allow-remove-essential - sudo apt-get update - sudo apt upgrade - sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq - sudo apt-get clean - sudo dmesg -c > /var/tmp/dmesg-prerun - echo "::endgroup::" -} - -function mod_build() { - echo "::group::Generate debian packages" - ./autogen.sh - ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan - make --no-print-directory --silent native-deb-utils native-deb-kmod - mv ../*.deb . - rm ./openzfs-zfs-dracut*.deb ./openzfs-zfs-dkms*.deb - echo "$ImageOS-$ImageVersion" > tests/ImageOS.txt - echo "::endgroup::" -} - -function mod_install() { - # install the pre-built module only on the same runner image - MOD=`cat tests/ImageOS.txt` - if [ "$MOD" != "$ImageOS-$ImageVersion" ]; then - rm -f *.deb - mod_build - fi - - echo "::group::Install and load modules" - # don't use kernel-shipped zfs modules - sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf - sudo apt-get install --fix-missing ./*.deb - - # Native Debian packages enable and start the services - # Stop zfs-zed daemon, as it may interfere with some ZTS test cases - sudo systemctl stop zfs-zed - sudo depmod -a - sudo modprobe zfs - sudo dmesg - sudo dmesg -c > /var/tmp/dmesg-module-load - echo "::endgroup::" - - echo "::group::Report CPU information" - lscpu - cat /proc/spl/kstat/zfs/chksum_bench - echo "::endgroup::" - - echo "::group::Optimize storage for ZFS testings" - # remove swap and umount fast storage - # 89GiB -> rootfs + bootfs with ~80MB/s -> don't care - # 64GiB -> /mnt with 420MB/s -> new testing ssd - sudo swapoff -a - - # this one is fast and mounted @ /mnt - # -> we reformat with ext4 + move it to /var/tmp - DEV="/dev/disk/azure/resource-part1" - sudo umount /mnt - sudo mkfs.ext4 -O ^has_journal -F $DEV - sudo mount -o noatime,barrier=0 $DEV /var/tmp - sudo chmod 1777 /var/tmp - - # disk usage afterwards - sudo df -h / - sudo df -h /var/tmp - sudo fstrim -a - echo "::endgroup::" -} - -case "$1" in - build) - prerun - mod_build - ;; - tests) - prerun - mod_install - ;; -esac diff --git a/.github/workflows/scripts/setup-functional.sh b/.github/workflows/scripts/setup-functional.sh deleted file mode 100755 index 08c4d872abdf..000000000000 --- a/.github/workflows/scripts/setup-functional.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -TDIR="/usr/share/zfs/zfs-tests/tests/functional" -echo -n "TODO=" -case "$1" in - part1) - # ~1h 20m - echo "cli_root" - ;; - part2) - # ~1h - ls $TDIR|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ',' - ;; - part3) - # ~1h - ls $TDIR|grep '^[n-qs-z]'|xargs|tr -s ' ' ',' - ;; - part4) - # ~1h - ls $TDIR|grep '^r'|xargs|tr -s ' ' ',' - ;; -esac diff --git a/.github/workflows/zfs-linux-tests.yml b/.github/workflows/zfs-linux-tests.yml deleted file mode 100644 index 753f3cd0214e..000000000000 --- a/.github/workflows/zfs-linux-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: zfs-linux-tests - -on: - workflow_call: - inputs: - os: - description: 'The ubuntu version: 20.02 or 22.04' - required: true - type: string - -jobs: - - zloop: - runs-on: ubuntu-${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v4 - with: - name: modules-${{ inputs.os }} - - name: Install modules - run: | - tar xzf modules-${{ inputs.os }}.tgz - .github/workflows/scripts/setup-dependencies.sh tests - - name: Tests - timeout-minutes: 30 - run: | - sudo mkdir -p /var/tmp/zloop - # run for 10 minutes or at most 2 iterations for a maximum runner - # time of 20 minutes. - sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60 - - name: Prepare artifacts - if: failure() - run: | - sudo chmod +r -R /var/tmp/zloop/ - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: Zpool-logs-${{ inputs.os }} - path: | - /var/tmp/zloop/*/ - !/var/tmp/zloop/*/vdev/ - retention-days: 14 - if-no-files-found: ignore - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: Zpool-files-${{ inputs.os }} - path: | - /var/tmp/zloop/*/vdev/ - retention-days: 14 - if-no-files-found: ignore - - sanity: - runs-on: ubuntu-${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v4 - with: - name: modules-${{ inputs.os }} - - name: Install modules - run: | - tar xzf modules-${{ inputs.os }}.tgz - .github/workflows/scripts/setup-dependencies.sh tests - - name: Tests - timeout-minutes: 60 - shell: bash - run: | - set -o pipefail - /usr/share/zfs/zfs-tests.sh -vKR -s 3G -r sanity | scripts/zfs-tests-color.sh - - name: Prepare artifacts - if: success() || failure() - run: | - RESPATH="/var/tmp/test_results" - mv -f $RESPATH/current $RESPATH/testfiles - tar cf $RESPATH/sanity.tar -h -C $RESPATH testfiles - - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: Logs-${{ inputs.os }}-sanity - path: /var/tmp/test_results/sanity.tar - if-no-files-found: ignore - - functional: - runs-on: ubuntu-${{ inputs.os }} - strategy: - fail-fast: false - matrix: - tests: [ part1, part2, part3, part4 ] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/download-artifact@v4 - with: - name: modules-${{ inputs.os }} - - name: Install modules - run: | - tar xzf modules-${{ inputs.os }}.tgz - .github/workflows/scripts/setup-dependencies.sh tests - - name: Setup tests - run: | - .github/workflows/scripts/setup-functional.sh ${{ matrix.tests }} >> $GITHUB_ENV - - name: Tests - timeout-minutes: 120 - shell: bash - run: | - set -o pipefail - /usr/share/zfs/zfs-tests.sh -vKR -s 3G -T ${{ env.TODO }} | scripts/zfs-tests-color.sh - - name: Prepare artifacts - if: success() || failure() - run: | - RESPATH="/var/tmp/test_results" - mv -f $RESPATH/current $RESPATH/testfiles - tar cf $RESPATH/${{ matrix.tests }}.tar -h -C $RESPATH testfiles - - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: Logs-${{ inputs.os }}-functional-${{ matrix.tests }} - path: /var/tmp/test_results/${{ matrix.tests }}.tar - if-no-files-found: ignore diff --git a/.github/workflows/zfs-linux.yml b/.github/workflows/zfs-linux.yml deleted file mode 100644 index e6b705c86055..000000000000 --- a/.github/workflows/zfs-linux.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: zfs-linux - -on: - push: - pull_request: - -jobs: - - build: - name: Build - strategy: - fail-fast: false - matrix: - os: [20.04, 22.04] - runs-on: ubuntu-${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Build modules - run: .github/workflows/scripts/setup-dependencies.sh build - - name: Prepare modules upload - run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt - - uses: actions/upload-artifact@v4 - with: - name: modules-${{ matrix.os }} - path: modules-${{ matrix.os }}.tgz - retention-days: 14 - - testings: - name: Testing - strategy: - fail-fast: false - matrix: - os: [20.04, 22.04] - needs: build - uses: ./.github/workflows/zfs-linux-tests.yml - with: - os: ${{ matrix.os }} - - cleanup: - if: always() - name: Cleanup - runs-on: ubuntu-22.04 - needs: testings - steps: - - uses: actions/download-artifact@v4 - - name: Generating summary - run: | - tar xzf modules-22.04/modules-22.04.tgz .github tests - .github/workflows/scripts/generate-summary.sh - # up to 4 steps, each can have 1 MiB output (for debugging log files) - - name: Summary for errors #1 - run: .github/workflows/scripts/generate-summary.sh 1 - - name: Summary for errors #2 - run: .github/workflows/scripts/generate-summary.sh 2 - - name: Summary for errors #3 - run: .github/workflows/scripts/generate-summary.sh 3 - - name: Summary for errors #4 - run: .github/workflows/scripts/generate-summary.sh 4 - - uses: actions/upload-artifact@v4 - with: - name: Summary Files - path: Summary/ diff --git a/.github/workflows/zfs-qemu.yml b/.github/workflows/zfs-qemu.yml new file mode 100644 index 000000000000..52b972d13a7a --- /dev/null +++ b/.github/workflows/zfs-qemu.yml @@ -0,0 +1,179 @@ +name: zfs-qemu + +on: + push: + pull_request: + +jobs: + + qemu-vm: + name: QEMU + strategy: + fail-fast: false + matrix: + os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd14, freebsd15, ubuntu22, ubuntu24] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Setup QEMU + timeout-minutes: 10 + run: .github/workflows/scripts/qemu-1-setup.sh + + - name: Start build machine + timeout-minutes: 5 + run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }} + + - name: Install dependencies + timeout-minutes: 20 + run: | + echo "Install dependencies in QEMU machine" + echo "ConnectTimeout 1" >> $HOME/.ssh/config + echo "StrictHostKeyChecking no" >> $HOME/.ssh/config + while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do + ssh 2>/dev/null zfs@192.168.122.10 "uname -a" && break + done + scp .github/workflows/scripts/qemu-3-deps.sh zfs@192.168.122.10:qemu-3-deps.sh + PID=`pidof /usr/bin/qemu-system-x86_64` + ssh zfs@192.168.122.10 '$HOME/qemu-3-deps.sh' ${{ matrix.os }} + # wait for poweroff to succeed + tail --pid=$PID -f /dev/null + sleep 5 # avoid this: "error: Domain is already active" + + - name: Build modules + timeout-minutes: 30 + run: | + echo "Build modules in QEMU machine" + sudo virsh start openzfs + while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do + ssh 2>/dev/null zfs@192.168.122.10 "uname -a" && break + done + rsync -ar $HOME/work/zfs/zfs zfs@192.168.122.10:./ + ssh zfs@192.168.122.10 '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }} + + - name: Setup testing machines + timeout-minutes: 5 + run: | + .github/workflows/scripts/qemu-5-setup.sh + # Save the VM's serial output (ttyS0) to /var/tmp/console.txt + # - ttyS0 on the VM corresponds to a local /dev/pty/N entry + # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry + RESPATH="/var/tmp/test_results" + mkdir -p $RESPATH/vm{1,2,3} + sudo mkdir -p /var/tmp + read "pty1" <<< $(sudo virsh ttyconsole "vm1") + read "pty2" <<< $(sudo virsh ttyconsole "vm2") + read "pty3" <<< $(sudo virsh ttyconsole "vm3") + sudo nohup bash -c "cat "$pty1" > $RESPATH/vm1/console.txt" & + sudo nohup bash -c "cat "$pty2" > $RESPATH/vm2/console.txt" & + sudo nohup bash -c "cat "$pty3" > $RESPATH/vm3/console.txt" & + echo "Console logging for $pty1, $pty2 and $pty3 started." + + - name: Run tests + timeout-minutes: 210 + run: | + .github/workflows/scripts/qemu-6-tests.sh + + - name: Test reports + timeout-minutes: 10 + run: | + .github/workflows/scripts/qemu-7-reports.sh + + - name: Prepare artifacts + if: success() || failure() + run: | + RESPATH="/var/tmp/test_results" + rsync -arL zfs@192.168.122.11:$RESPATH/current $RESPATH/vm1 || true + rsync -arL zfs@192.168.122.12:$RESPATH/current $RESPATH/vm2 || true + rsync -arL zfs@192.168.122.13:$RESPATH/current $RESPATH/vm3 || true + scp zfs@192.168.122.11:"/var/tmp/*.txt" $RESPATH/vm1 || true + scp zfs@192.168.122.12:"/var/tmp/*.txt" $RESPATH/vm2 || true + scp zfs@192.168.122.13:"/var/tmp/*.txt" $RESPATH/vm3 || true + cp -f /var/tmp/*.txt $RESPATH || true + ls -la /var/tmp + tar cf qemu-${{ matrix.os }}.tar -C $RESPATH -h . || true + + - uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: Logs-functional-${{ matrix.os }} + path: qemu-${{ matrix.os }}.tar + if-no-files-found: ignore + + cleanup: + if: always() + name: Cleanup + runs-on: ubuntu-latest + needs: [ qemu-vm ] + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/download-artifact@v4 + - name: Generating summary + run: .github/workflows/scripts/generate-summary.sh + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 1 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 2 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 3 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 4 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 5 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 6 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 7 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 8 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 9 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 10 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 11 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 12 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 13 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 14 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 15 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 16 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 17 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 18 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 19 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 20 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 21 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 22 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 23 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 24 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 25 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 26 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 27 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 28 + - name: Generating summary... + run: .github/workflows/scripts/generate-summary.sh 29 + - uses: actions/upload-artifact@v4 + with: + name: Summary Files + path: out-* diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh index 85854085f560..b5d8a4fc32ae 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh @@ -67,6 +67,7 @@ log_onexit cleanup cleanup + typeset -i i=0 while (( i < ${#shareopts[*]} )) do diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh index ec11610742fb..e7588e207b46 100755 --- a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh @@ -68,6 +68,7 @@ log_must mount_redacted -f $recvfs # when we re-enable redaction blkptrs. # #log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1 +log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt" log_must diff $send_mnt/f2 $recv_mnt/f2 log_must zfs rollback -R $clone@snap log_must zfs destroy -R $recvfs @@ -96,6 +97,7 @@ log_must zfs destroy -R $clone2 log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream" log_must eval "zfs recv $recvfs <$stream" log_must mount_redacted -f $recvfs +log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt" log_must [ "$(ls $send_mnt)" == "$(ls $recv_mnt)" ] log_must zfs destroy -R $recvfs log_must zfs rollback -R $sendfs@snap