diff --git a/.github/workflows/scripts/README.md b/.github/workflows/scripts/README.md new file mode 100644 index 000000000000..5b2ba703e408 --- /dev/null +++ b/.github/workflows/scripts/README.md @@ -0,0 +1,9 @@ + +Workflow for each operating system: +- install qemu on the github runner +- download current cloud image of operating system +- start and init that image via cloud-init +- install dependencies and poweroff system +- start system and build openzfs and then poweroff again +- clone build system and start 3 instances of it +- the functional testings complete within times < 3h diff --git a/.github/workflows/scripts/merge_summary.awk b/.github/workflows/scripts/merge_summary.awk new file mode 100755 index 000000000000..20370463e2ed --- /dev/null +++ b/.github/workflows/scripts/merge_summary.awk @@ -0,0 +1,104 @@ +#!/bin/awk -f +# +# Merge multiple ZTS tests results summaries into a single summary. This is +# needed when you're running different parts of ZTS on different tests +# runners or VMs. +# +# Usage: +# +# ./merge_summary.awk summary1.txt [summary2.txt] [summary3.txt] ... +# +# or: +# +# cat summary*.txt | ./merge_summary.awk +# +BEGIN { + i=-1 + pass=0 + fail=0 + skip=0 + state="" + cl=0 + el=0 + upl=0 + ul=0 + + # Total seconds of tests runtime + total=0; +} + +# Skip empty lines +/^\s*$/{next} + +# Skip Configuration and Test lines +/^Test:/{state=""; next} +/Configuration/{state="";next} + +# When we see "test-runner.py" stop saving config lines, and +# save test runner lines +/test-runner.py/{state="testrunner"; runner=runner$0"\n"; next} + +# We need to differentiate the PASS counts from test result lines that start +# with PASS, like: +# +# PASS mv_files/setup +# +# Use state="pass_count" to differentiate +# +/Results Summary/{state="pass_count"; next} +/PASS/{ if (state=="pass_count") {pass += $2}} +/FAIL/{ if (state=="pass_count") {fail += $2}} +/SKIP/{ if (state=="pass_count") {skip += $2}} +/Running Time/{ + state=""; + running[i]=$3; + split($3, arr, ":") + total += arr[1] * 60 * 60; + total += arr[2] * 60; + total += arr[3] + next; +} + +/Tests with results other than PASS that are expected/{state="expected_lines"; next} +/Tests with result of PASS that are unexpected/{state="unexpected_pass_lines"; next} +/Tests with results other than PASS that are unexpected/{state="unexpected_lines"; next} +{ + if (state == "expected_lines") { + expected_lines[el] = $0 + el++ + } + + if (state == "unexpected_pass_lines") { + unexpected_pass_lines[upl] = $0 + upl++ + } + if (state == "unexpected_lines") { + unexpected_lines[ul] = $0 + ul++ + } +} + +# Reproduce summary +END { + print runner; + print "\nResults Summary" + print "PASS\t"pass + print "FAIL\t"fail + print "SKIP\t"skip + print "" + print "Running Time:\t"strftime("%T", total, 1) + if (pass+fail+skip > 0) { + percent_passed=(pass/(pass+fail+skip) * 100) + } + printf "Percent passed:\t%3.2f%", percent_passed + + print "\n\nTests with results other than PASS that are expected:" + for (j in expected_lines) + print expected_lines[j] + print "\n\nTests with result of PASS that are unexpected:" + for (j in unexpected_pass_lines) + print unexpected_pass_lines[j] + print "\n\nTests with results other than PASS that are unexpected:" + for (j in unexpected_lines) + print unexpected_lines[j] +} diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh new file mode 100755 index 000000000000..c371b71f55d3 --- /dev/null +++ b/.github/workflows/scripts/qemu-1-setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +###################################################################### +# 1) setup qemu instance on action runner +###################################################################### + +set -eu + +# install needed packages +sudo apt-get update +sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \ + ksmtuned virt-manager linux-modules-extra-`uname -r` + +# generate ssh keys +rm -f ~/.ssh/id_ed25519 +ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N "" + +# no need for some scheduler +for i in /sys/block/s*/queue/scheduler; do + echo "none" | sudo tee $i > /dev/null +done + +# this one is fast and mostly free +sudo mount -o remount,rw,noatime,barrier=0 /mnt + +# we expect RAM shortage +cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null +KSM_MONITOR_INTERVAL=60 + +# Millisecond sleep between ksm scans for 16Gb server. +# Smaller servers sleep more, bigger sleep less. +KSM_SLEEP_MSEC=10 +KSM_NPAGES_BOOST=300 +KSM_NPAGES_DECAY=-50 +KSM_NPAGES_MIN=64 +KSM_NPAGES_MAX=2048 + +KSM_THRES_COEF=20 +KSM_THRES_CONST=2048 + +LOGFILE=/var/log/ksmtuned.log +DEBUG=1 +EOF + +sudo systemctl restart ksm +sudo systemctl restart ksmtuned diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh new file mode 100755 index 000000000000..137bf6f58aa5 --- /dev/null +++ b/.github/workflows/scripts/qemu-2-start.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash + +###################################################################### +# 2) start qemu with some operating system, init via cloud-init +###################################################################### + +set -eu + +# short name used in zfs-qemu.yml +OS="$1" + +# OS variant (virt-install --os-variant list) +OSv=$OS + +# compressed with .zst extension +FREEBSD="https://github.com/mcmilk/openzfs-freebsd-images/releases/download/v2024-08-10" +URLzs="" + +# Ubuntu mirrors +#UBMIRROR="https://cloud-images.ubuntu.com" +#UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images" +UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images" + +case "$OS" in + almalinux8) + OSNAME="AlmaLinux 8" + URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2" + ;; + almalinux9) + OSNAME="AlmaLinux 9" + URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2" + ;; + archlinux) + OSNAME="Archlinux" + URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2" + ;; + centos-stream9) + OSNAME="CentOS Stream 9" + URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" + ;; + debian11) + OSNAME="Debian 11" + URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2" + ;; + debian12) + OSNAME="Debian 12" + URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2" + ;; + fedora39) + OSNAME="Fedora 39" + OSv="fedora39" + URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2" + ;; + fedora40) + OSNAME="Fedora 40" + OSv="fedora39" + URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2" + ;; + freebsd13r) + OSNAME="FreeBSD 13.3-RELEASE" + OSv="freebsd13.0" + URLzs="$FREEBSD/amd64-freebsd-13.3-RELEASE.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + freebsd13) + OSNAME="FreeBSD 13.4-STABLE" + OSv="freebsd13.0" + URLzs="$FREEBSD/amd64-freebsd-13.4-STABLE.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + freebsd14r) + OSNAME="FreeBSD 14.1-RELEASE" + OSv="freebsd14.0" + URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + freebsd14) + OSNAME="FreeBSD 14.1-STABLE" + OSv="freebsd14.0" + URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + freebsd15) + OSNAME="FreeBSD 15.0-CURRENT" + OSv="freebsd14.0" + URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst" + BASH="/usr/local/bin/bash" + ;; + tumbleweed) + OSNAME="openSUSE Tumbleweed" + OSv="opensusetumbleweed" + MIRROR="http://opensuse-mirror-gce-us.susecloud.net" + URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2" + ;; + ubuntu20) + OSNAME="Ubuntu 20.04" + OSv="ubuntu20.04" + URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img" + ;; + ubuntu22) + OSNAME="Ubuntu 22.04" + OSv="ubuntu22.04" + URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img" + ;; + ubuntu24) + OSNAME="Ubuntu 24.04" + OSv="ubuntu24.04" + URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img" + ;; + *) + echo "Wrong value for OS variable!" + exit 111 + ;; +esac + +# freebsd15 -> used in zfs-qemu.yml +echo "$OS" > /var/tmp/os.txt + +# freebsd14.0 -> used for virt-install +echo "$OSv" > /var/tmp/osvariant.txt + +# FreeBSD 15 (Current) -> used for summary +echo "$OSNAME" > /var/tmp/osname.txt + +IMG="/mnt/tests/cloudimg.qcow2" +DISK="/mnt/tests/openzfs.qcow2" +sudo mkdir -p "/mnt/tests" +sudo chown $(whoami) /mnt/tests + +# we are downloading via axel, curl and wget are mostly slower and +# require more return value checking +if [ ! -z "$URLzs" ]; then + echo "Loading image $URLzs ..." + time axel -q -o "$IMG.zst" "$URLzs" + zstd -q -d --rm "$IMG.zst" +else + echo "Loading image $URL ..." + time axel -q -o "$IMG" "$URL" +fi + +# 256k cluster seems the best in terms of speed for now +qemu-img convert -q -f qcow2 -O qcow2 -c \ + -o compression_type=zstd,cluster_size=256k $IMG $DISK +rm -f $IMG + +echo "Resizing image to 60GiB ..." +qemu-img resize -q $DISK 60G + +PUBKEY=`cat ~/.ssh/id_ed25519.pub` +cat < /tmp/user-data +#cloud-config + +fqdn: $OS + +# user:zfs password:1 +users: +- name: root + shell: $BASH +- name: zfs + sudo: ALL=(ALL) NOPASSWD:ALL + shell: $BASH + lock-passwd: false + passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/ + ssh_authorized_keys: + - $PUBKEY + +growpart: + mode: auto + devices: ['/'] + ignore_growroot_disabled: false +EOF + +sudo virsh net-update default add ip-dhcp-host \ + "" --live --config + +# 12GiB RAM for building the module, TY Github :) +sudo virt-install \ + --os-variant $OSv \ + --name "openzfs" \ + --cpu host-passthrough \ + --virt-type=kvm --hvm \ + --vcpus=4,sockets=1 \ + --memory $((1024*12)) \ + --memballoon model=virtio \ + --graphics none \ + --network bridge=virbr0,model=e1000,mac='52:54:00:83:79:00' \ + --cloud-init user-data=/tmp/user-data \ + --disk $DISK,bus=virtio,cache=none,format=qcow2,driver.discard=unmap \ + --import --noautoconsole >/dev/null + +# in case the directory isn't there already +mkdir -p $HOME/.ssh + +cat <> $HOME/.ssh/config +# no questions please +StrictHostKeyChecking no + +# small timeout, used in while loops later +ConnectTimeout 1 +EOF diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh new file mode 100755 index 000000000000..d890861066a9 --- /dev/null +++ b/.github/workflows/scripts/qemu-3-deps.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +###################################################################### +# 3) install dependencies for compiling and loading +###################################################################### + +set -eu + +function archlinux() { + echo "##[group]Running pacman -Syu" + sudo pacman -Syu --noconfirm + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \ + fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils parted \ + pax perf python-packaging python-setuptools qemu-guest-agent ksh samba \ + sysstat rng-tools rsync wget + echo "##[endgroup]" +} + +function debian() { + export DEBIAN_FRONTEND="noninteractive" + + echo "##[group]Running apt-get update+upgrade" + sudo apt-get update -y + sudo apt-get upgrade -y + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo apt-get install -y \ + acl alien attr autoconf bc cpio curl dbench dh-python \ + dkms fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq \ + libacl1-dev libaio-dev libattr1-dev libblkid-dev \ + libcurl4-openssl-dev libdevmapper-dev libelf-dev libffi-dev \ + libmount-dev libpam0g-dev libselinux-dev libssl-dev libtool \ + libtool-bin libudev-dev linux-headers-$(uname -r) lsscsi \ + nfs-kernel-server pamtester parted python3 python3-all-dev \ + python3-cffi python3-dev python3-distlib python3-packaging \ + python3-setuptools python3-sphinx qemu-guest-agent rng-tools \ + rpm2cpio rsync samba sysstat uuid-dev watchdog wget xfslibs-dev \ + zlib1g-dev + echo "##[endgroup]" +} + +function freebsd() { + export ASSUME_ALWAYS_YES="YES" + + echo "##[group]Install Development Tools" + sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \ + gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \ + pkgconf python python3 pamtester pamtester qemu-guest-agent rsync + sudo pkg install -xy \ + '^samba4[[:digit:]]+$' \ + '^py3[[:digit:]]+-cffi$' \ + '^py3[[:digit:]]+-sysctl$' \ + '^py3[[:digit:]]+-packaging$' + echo "##[endgroup]" +} + +# common packages for: almalinux, centos, redhat +function rhel() { + echo "##[group]Running dnf update" + echo "max_parallel_downloads=10" | sudo -E tee -a /etc/dnf/dnf.conf + sudo dnf clean all + sudo dnf update -y --setopt=fastestmirror=1 --refresh + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo dnf group install -y "Development Tools" + sudo dnf install -y \ + acl attr bc bzip2 curl dbench dkms elfutils-libelf-devel fio gdb git \ + jq kernel-rpm-macros ksh libacl-devel libaio-devel libargon2-devel \ + libattr-devel libblkid-devel libcurl-devel libffi-devel ncompress \ + libselinux-devel libtirpc-devel libtool libudev-devel libuuid-devel \ + lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \ + python3 python3-cffi python3-devel python3-packaging kernel-devel \ + python3-setuptools qemu-guest-agent rng-tools rpcgen rpm-build rsync \ + samba sysstat systemd watchdog wget xfsprogs-devel zlib-devel + echo "##[endgroup]" +} + +function tumbleweed() { + echo "##[group]Running zypper is TODO!" + sleep 23456 + echo "##[endgroup]" +} + +# Install dependencies +case "$1" in + almalinux8) + echo "##[group]Enable epel and powertools repositories" + sudo dnf config-manager -y --set-enabled powertools + sudo dnf install -y epel-release + echo "##[endgroup]" + rhel + echo "##[group]Install kernel-abi-whitelists" + sudo dnf install -y kernel-abi-whitelists + echo "##[endgroup]" + ;; + almalinux9|centos-stream9) + echo "##[group]Enable epel and crb repositories" + sudo dnf config-manager -y --set-enabled crb + sudo dnf install -y epel-release + echo "##[endgroup]" + rhel + echo "##[group]Install kernel-abi-stablelists" + sudo dnf install -y kernel-abi-stablelists + echo "##[endgroup]" + ;; + archlinux) + archlinux + ;; + debian*) + debian + echo "##[group]Install Debian specific" + sudo apt-get install -yq linux-perf dh-sequence-dkms + echo "##[endgroup]" + ;; + fedora*) + rhel + ;; + freebsd*) + freebsd + ;; + tumbleweed) + tumbleweed + ;; + ubuntu*) + debian + echo "##[group]Install Ubuntu specific" + sudo apt-get install -yq linux-tools-common libtirpc-dev \ + linux-modules-extra-$(uname -r) + if [ "$1" != "ubuntu20" ]; then + sudo apt-get install -yq dh-sequence-dkms + fi + echo "##[endgroup]" + echo "##[group]Delete Ubuntu OpenZFS modules" + for i in `find /lib/modules -name zfs -type d`; do sudo rm -rvf $i; done + echo "##[endgroup]" + ;; +esac + +# Start services +echo "##[group]Enable services" +case "$1" in + freebsd*) + # add virtio things + echo 'virtio_load="YES"' | sudo -E tee -a /boot/loader.conf + for i in balloon blk console random scsi; do + echo "virtio_${i}_load=\"YES\"" | sudo -E tee -a /boot/loader.conf + done + echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab + sudo -E mount /dev/fd + sudo -E touch /etc/zfs/exports + sudo -E sysrc mountd_flags="/etc/zfs/exports" + echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null + sudo -E service nfsd enable + sudo -E service qemu-guest-agent enable + sudo -E service samba_server enable + ;; + debian*|ubuntu*) + sudo -E systemctl enable nfs-kernel-server + sudo -E systemctl enable qemu-guest-agent + sudo -E systemctl enable smbd + ;; + *) + # All other linux distros + sudo -E systemctl enable nfs-server + sudo -E systemctl enable qemu-guest-agent + sudo -E systemctl enable smb + ;; +esac +echo "##[endgroup]" + +# Enable serial console and remove 'quiet' from linux kernel cmdline +case "$1" in + freebsd*) + # console is turned on @ FreeBSD images from here: + # https://github.com/mcmilk/zfs/tree/master + true + ;; + *) + # should fit for most distros, may be optimized a bit later + echo "##[group]Enable serial output" + sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0,115200n8 random.trust_cpu=on/g; s/quiet //g' /etc/default/grub || true + for i in /boot/grub/grub.cfg /etc/grub2.cfg /etc/grub2-efi.cfg /boot/grub2/grub.cfg ; do + test -e $i || continue + echo sudo grub-mkconfig -o $i + sudo grub-mkconfig -o $i + done + echo "##[endgroup]" + ;; +esac + +# reset cloud-init configuration and poweroff +sudo cloud-init clean --logs +sleep 2 && sudo poweroff & +exit 0 diff --git a/.github/workflows/scripts/qemu-4-build.sh b/.github/workflows/scripts/qemu-4-build.sh new file mode 100755 index 000000000000..cabfc6c487d1 --- /dev/null +++ b/.github/workflows/scripts/qemu-4-build.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +###################################################################### +# 4) configure and build openzfs modules +###################################################################### + +set -eu + +function run() { + LOG="/var/tmp/build-stderr.txt" + echo "**************************************************" + echo "`date` ($*)" + echo "**************************************************" + $@ 2>>$LOG +} + +function freebsd() { + export MAKE="gmake" + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr/local \ + --with-libintl-prefix=/usr/local \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run gmake -j`sysctl -n hw.ncpu` + echo "##[endgroup]" + + echo "##[group]Install" + run sudo gmake install + echo "##[endgroup]" +} + +function linux() { + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run make -j$(nproc) + echo "##[endgroup]" + + echo "##[group]Install" + run sudo make install + echo "##[endgroup]" +} + +function rpm_build_and_install() { + EXTRA_CONFIG="${1:-}" + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG + echo "##[endgroup]" + + echo "##[group]Build" + run make pkg-kmod pkg-utils + echo "##[endgroup]" + + echo "##[group]Install" + run sudo yum -y --skip-broken localinstall $(ls *.rpm | grep -v src.rpm) + echo "##[endgroup]" + +} + +function deb_build_and_install() { +echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run make native-deb-kmod native-deb-utils + echo "##[endgroup]" + + echo "##[group]Install" + # Do kmod install. Note that when you build the native debs, the + # packages themselves are placed in parent directory '../' rather than + # in the source directory like the rpms are. + run sudo apt-get -y install `find ../ | grep -E '\.deb$' | grep -Ev 'dkms|dracut'` + echo "##[endgroup]" +} + +# Debug: show kernel cmdline +if [ -e /proc/cmdline ] ; then + cat /proc/cmdline || true +fi + +cd $HOME/zfs +export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" + +# build +case "$1" in + freebsd*) + freebsd + ;; + alma*|centos*) + rpm_build_and_install "--with-spec=redhat" + ;; + fedora*) + rpm_build_and_install + ;; + debian*|ubuntu*) + deb_build_and_install + ;; + *) + linux + ;; +esac + +# save some sysinfo +uname -a > /var/tmp/uname.txt + +# reset cloud-init configuration and poweroff +sudo cloud-init clean --logs +sleep 2 && sudo poweroff & +exit 0 diff --git a/.github/workflows/scripts/qemu-5-setup.sh b/.github/workflows/scripts/qemu-5-setup.sh new file mode 100755 index 000000000000..1e18be84ecba --- /dev/null +++ b/.github/workflows/scripts/qemu-5-setup.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +###################################################################### +# 5) start test machines and load openzfs module +###################################################################### + +set -eu + +# wait for poweroff to succeed +PID=`pidof /usr/bin/qemu-system-x86_64` +tail --pid=$PID -f /dev/null +sudo virsh undefine openzfs + +PUBKEY=`cat ~/.ssh/id_ed25519.pub` +OSv=`cat /var/tmp/osvariant.txt` +OS=`cat /var/tmp/os.txt` + +# definition of ressources per operating system +case "$OS" in + freebsd*) + # 2x CPU=4 RAM=6 -> FreeBSD 13 (2h 10m) + # 2x CPU=4 RAM=6 -> FreeBSD 14 (2h 10m) + VMs=2 + CPU=4 + RAM=6 + ;; + *) + # 2x CPU=4 RAM=7 -> Almalinux 8 (3h 12m) + # 2x CPU=4 RAM=7 -> CentOS 9 (3h 2m) + # 2x CPU=4 RAM=7 -> Debian 11 (3h 11m) + # 2x CPU=4 RAM=7 -> Ubuntu 20 (2h 49m) + # 2x CPU=4 RAM=7 -> Ubuntu 22 (3h 26m) + # 2x CPU=4 RAM=7 -> Ubuntu 24 (3h 10m) + # 2x CPU=4 RAM=7 -> Fedora40 (3h 33m) + VMs=2 + CPU=4 + RAM=7 + ;; +esac + +# this can be different for each distro +echo $VMs > /var/tmp/vms.txt + +# setup the testing vm's +for i in `seq 1 $VMs`; do + echo "Generating disk for vm$i..." + sudo qemu-img create -q -f qcow2 -F qcow2 \ + -o compression_type=zstd,cluster_size=256k \ + -b /mnt/tests/openzfs.qcow2 "/mnt/tests/vm$i.qcow2" + + cat < /tmp/user-data +#cloud-config + +fqdn: vm$i + +# user:zfs password:1 +users: +- name: root + shell: $BASH +- name: zfs + sudo: ALL=(ALL) NOPASSWD:ALL + shell: $BASH + lock-passwd: false + passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/ + ssh_authorized_keys: + - $PUBKEY + +growpart: + mode: auto + devices: ['/'] + ignore_growroot_disabled: false +EOF + + sudo virsh net-update default add ip-dhcp-host \ + "" --live --config + + sudo virt-install \ + --os-variant $OSv \ + --name "vm$i" \ + --cpu host-passthrough \ + --virt-type=kvm --hvm \ + --vcpus=$CPU,sockets=1 \ + --memory $((1024*RAM)) \ + --memballoon model=virtio \ + --graphics none \ + --cloud-init user-data=/tmp/user-data \ + --network bridge=virbr0,model=e1000,mac="52:54:00:83:79:0$i" \ + --disk /mnt/tests/vm$i.qcow2,bus=virtio,cache=none,format=qcow2,driver.discard=unmap \ + --import --noautoconsole >/dev/null +done + +# setup cron job on the host side +case "$OS" in + freebsd*) + true + ;; + *) + # Linux based systems, trim the qcow2 files + echo "exec 1>/dev/null 2>/dev/null" > cronjob.sh + for i in `seq 1 $VMs`; do + echo "virsh domfstrim vm$i" >> cronjob.sh + done + echo "fstrim /mnt" >> cronjob.sh + sudo chmod +x cronjob.sh + sudo mv -f cronjob.sh /root/cronjob.sh + echo '*/20 * * * * /root/cronjob.sh' > crontab.txt + sudo crontab crontab.txt + rm crontab.txt + ;; +esac + +# check if the machines are okay +echo "Waiting for vm's to come up... (${VMs}x CPU=$CPU RAM=$RAM)" +for i in `seq 1 $VMs`; do + while true; do + ssh 2>/dev/null zfs@192.168.122.1$i "uname -a" && break + done +done +echo "All $VMs VMs are up now." + +# Save the VM's serial output (ttyS0) to /var/tmp/console.txt +# - ttyS0 on the VM corresponds to a local /dev/pty/N entry +# - use 'virsh ttyconsole' to lookup the /dev/pty/N entry +RESPATH="/var/tmp/test_results" +for i in `seq 1 $VMs`; do + mkdir -p $RESPATH/vm$i + read "pty" <<< $(sudo virsh ttyconsole vm$i) + sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" & +done +echo "Console logging for ${VMs}x $OS started." diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh new file mode 100755 index 000000000000..6d69a70309c5 --- /dev/null +++ b/.github/workflows/scripts/qemu-6-tests.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +###################################################################### +# 6) load openzfs module and run the tests +# +# called on runner: qemu-6-tests.sh (without extra args) +# called on qemu-vm: qemu-6-tests.sh $OS $2/$3 +###################################################################### + +set -o pipefail + +# called directly on the runner +if [ -f /var/tmp/vms.txt ]; then + cd "/var/tmp" + + OS=`cat os.txt` + VMs=`cat vms.txt` + SSH=`which ssh` + + BASE="$HOME/work/zfs/zfs" + TESTS='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh' + COLOR="$BASE/scripts/zfs-tests-color.sh" + + # df statistics - keep an eye on disk usage + echo "Disk usage before:" > disk-usage.txt + df -h /mnt/tests >> disk-usage.txt + + for i in `seq 1 $VMs`; do + IP="192.168.122.1$i" + daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \ + $SSH zfs@$IP $TESTS $OS $i $VMs + # give us the output of stdout + stderr - with prefix ;) + tail -fq vm${i}log.txt | $COLOR | sed -e "s/^/vm${i}: /g" & + echo $! > vm${i}log.pid + done + + # wait for all vm's to finish + for i in `seq 1 $VMs`; do + tail --pid=`cat vm${i}.pid` -f /dev/null + pid=`cat vm${i}log.pid` + rm -f vm${i}log.pid + kill $pid + done + + # df statistics part 2 + echo "Disk usage afterwards:" >> disk-usage.txt + df -h /mnt/tests >> disk-usage.txt + echo "VM files take this space:" >> disk-usage.txt + du -sh /mnt/tests >> disk-usage.txt + + exit 0 +fi + +# this part runs inside qemu vm +export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin" +case "$1" in + freebsd*) + # when freebsd's zfs is loaded, unload this one + sudo kldstat -n zfs 2>/dev/null && sudo kldunload zfs + sudo -E ./zfs/scripts/zfs.sh + sudo dmesg -c > /var/tmp/dmesg-prerun.txt + TDIR="/usr/local/share/zfs" + ;; + *) + sudo -E modprobe zfs + sudo dmesg -c > /var/tmp/dmesg-prerun.txt + TDIR="/usr/share/zfs" + ;; +esac + +# run functional testings +TAGS=$2/$3 +#TAGS=casenorm,zpool_trim,trim + +$TDIR/zfs-tests.sh -vK -s 3G -T $TAGS +RV=$? + +# we wont fail here, this will be done later +echo $RV > /var/tmp/exitcode.txt +exit 0 diff --git a/.github/workflows/scripts/qemu-7-prepare.sh b/.github/workflows/scripts/qemu-7-prepare.sh new file mode 100755 index 000000000000..5f7ca4cc8c5c --- /dev/null +++ b/.github/workflows/scripts/qemu-7-prepare.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +###################################################################### +# 7) output the results of the previous stage in an ordered way +###################################################################### + +set -o pipefail + +cd /var/tmp +OS=`cat os.txt` + +# helper function for showing some content with headline +function showfile() { + content=`cat $1` + hl="$2" + + if [ $3=KB ]; then + SIZE=`stat --printf="%s" "$file"` + SIZE=$((SIZE/1024)) + kb=" ($SIZE KiB)" + else + kb="" + fi +cat < tmp$$ +##[group]$hl${kb} +$content +##[endgroup] +EOF + cat tmp$$ + rm -f tmp$$ +} + +# check if building the module has failed +RESPATH="/var/tmp/test_results" +if [ ! -s vms.txt ]; then + mkdir -p $RESPATH + cd $RESPATH + echo "!!! ZFS module didn't build successfully !!!" \ + | tee summary.txt | tee clean-summary.txt + scp zfs@192.168.122.10:"/var/tmp/*.txt" $RESPATH || true + cp -f /var/tmp/*.txt $RESPATH || true + tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true + exit 0 +fi + +# build was okay +VMs=`cat vms.txt` + +#################################################################### +# vm${N}log.txt -> output of ssh/tail -> for merged summary.txt +# +# vm${N}/build-stderr.txt -> two copies -> moved to one file +# vm${N}/dmesg-prerun.txt -> dmesg output of vm start +# vm${N}/console.txt -> serial output of vm +# vm${N}/uname.txt -> output of uname -a on test vm +# +# vm${N}/current/log -> if not there, kernel panic loading +# vm${N}/current/results -> if not there, kernel panic testings +#################################################################### + +BASE="$HOME/work/zfs/zfs" +MERGE="$BASE/.github/workflows/scripts/merge_summary.awk" + +# catch result files of testings +for i in `seq 1 $VMs`; do + rsync -arL zfs@192.168.122.1$i:$RESPATH/current $RESPATH/vm$i || true + scp zfs@192.168.122.1$i:"/var/tmp/*.txt" $RESPATH/vm$i || true +done +cp -f /var/tmp/*.txt $RESPATH || true +cd $RESPATH + +# Save a list of all failed test logs for easy access +awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; \ + /\[SKIP\]|\[PASS\]/{ show=0; } show' \ + vm*/current/log >> summary-failure-logs.txt + +$MERGE vm*log.txt > summary-clean.txt +$MERGE vm*log.txt | $BASE/scripts/zfs-tests-color.sh > summary.txt +cat summary.txt summary-failure-logs.txt >> summary-with-logs.txt + +# we should have vm-count identical build-stderr.txt files, need only one! +for i in `seq 1 $VMs`; do + file="vm$i/build-stderr.txt" + test -s "$file" && mv -f $file build-stderr.txt +done + +# artifact ready now +tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true + +# called for summary +if [ ! -z "$1" ]; then + file="build-stderr.txt" + test -s "$file" && showfile "$file" "Stderr of module build" "KB" + + for i in `seq 1 $VMs`; do + file="vm$i/dmesg-prerun.txt" + test -s "$file" && showfile "$file" "vm$i: dmesg kernel" "KB" + + file="vm$i/console.txt" + test -s "$file" && showfile "$file" "vm$i: serial console" "KB" + + file="vm${i}log.txt" + test -s $file && cat $file | $BASE/scripts/zfs-tests-color.sh > $file.x + test -s "$file.x" && showfile "$file.x" "vm$i: test results" "KB" + done +fi diff --git a/.github/workflows/scripts/qemu-8-summary.sh b/.github/workflows/scripts/qemu-8-summary.sh new file mode 100755 index 000000000000..789e9b09095a --- /dev/null +++ b/.github/workflows/scripts/qemu-8-summary.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +###################################################################### +# 8) generate github summary page of all the testings +###################################################################### + +# max size in KiB of debug output +DEBUG_MAX="300" + +function output() { + echo -e $* >> "out-$logfile.md" +} + +function outfile() { + CUR=`stat --printf="%s" "out-$logfile.md"` + ADD=`stat --printf="%s" "$1"` + X=$((CUR+ADD)) + if [ $X -gt $((1024*1023)) ]; then + logfile=$((logfile+1)) + fi + cat "$1" >> "out-$logfile.md" +} + +function send2github() { + test -f "$1" || exit 0 + dd if="$1" bs=1023k count=1 >> $GITHUB_STEP_SUMMARY +} + +# generate summary of one test +function generate() { + osname=`cat osname.txt` + VMs=`cat vms.txt` + + logfile=$((logfile+1)) + output "\n## Functional Tests: $osname\n" + for i in `seq 1 $VMs`; do + for f in uname.txt; do + test -s vm$i/$f && cat vm$i/$f >> $f + touch $f + done + done + + output "
"
+  outfile uname.txt
+  output "
" + + if [ -s "summary-clean.txt" ]; then + output "" + outfile "summary-clean.txt" + output "" + fi +} + +# https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits +# Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB. +# [ ] can not show all error findings here +# [x] split files into smaller ones and create additional steps + +# first call, generate all summaries +if [ ! -f out-1.md ]; then + # create ./zts-report.py for generate() + TEMPLATE="tests/test-runner/bin/zts-report.py.in" + cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py + chmod +x ./zts-report.py + + logfile="0" + for tarfile in Logs-functional-*/qemu-*.tar; do + if [ ! -s "$tarfile" ]; then + output "\n# Functional Tests: unknown\n" + output ":exclamation: Tarfile $tarfile is empty :exclamation:" + continue + fi + rm -rf vm* *.txt + tar xf "$tarfile" + generate + done + send2github out-1.md +else + send2github out-$1.md +fi diff --git a/.github/workflows/zfs-qemu.yml b/.github/workflows/zfs-qemu.yml new file mode 100644 index 000000000000..d68085667c9f --- /dev/null +++ b/.github/workflows/zfs-qemu.yml @@ -0,0 +1,174 @@ +name: zfs-qemu + +on: + push: + pull_request: + +jobs: + + qemu-vm: + name: qemu-vm + strategy: + fail-fast: false + matrix: + # all: + # os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24] + # openzfs: + os: [almalinux8, almalinux9, centos-stream9, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24] + # freebsd: + # os: [freebsd13, freebsd13r, freebsd14, freebsd14r] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Setup QEMU + timeout-minutes: 10 + run: .github/workflows/scripts/qemu-1-setup.sh + + - name: Start build machine + timeout-minutes: 10 + run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }} + + - name: Install dependencies + timeout-minutes: 20 + run: | + echo "Install dependencies in QEMU machine" + IP=192.168.122.10 + while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do + ssh 2>/dev/null zfs@$IP "uname -a" && break + done + scp .github/workflows/scripts/qemu-3-deps.sh zfs@$IP:qemu-3-deps.sh + PID=`pidof /usr/bin/qemu-system-x86_64` + ssh zfs@$IP '$HOME/qemu-3-deps.sh' ${{ matrix.os }} + # wait for poweroff to succeed + tail --pid=$PID -f /dev/null + sleep 5 # avoid this: "error: Domain is already active" + rm -f $HOME/.ssh/known_hosts + + - name: Build modules + timeout-minutes: 30 + run: | + echo "Build modules in QEMU machine" + sudo virsh start openzfs + IP=192.168.122.10 + while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do + ssh 2>/dev/null zfs@$IP "uname -a" && break + done + rsync -ar $HOME/work/zfs/zfs zfs@$IP:./ + ssh zfs@$IP '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }} + + - name: Setup testing machines + timeout-minutes: 5 + run: .github/workflows/scripts/qemu-5-setup.sh + + - name: Run tests + timeout-minutes: 270 + run: .github/workflows/scripts/qemu-6-tests.sh ${{ matrix.os }} + + - name: Prepare artifacts + if: always() + timeout-minutes: 10 + run: .github/workflows/scripts/qemu-7-prepare.sh + + - uses: actions/upload-artifact@v4 + id: artifact-upload + if: always() + with: + name: Logs-functional-${{ matrix.os }} + path: /tmp/qemu-${{ matrix.os }}.tar + if-no-files-found: ignore + + - name: Test Summary + if: success() || failure() + run: | + # overview + P=`pwd` + cd /var/tmp/test_results + cat summary.txt + echo "" + echo "Full logs for download:" + echo '${{ steps.artifact-upload.outputs.artifact-url }}' + echo "" + echo "File listing:" + ls -l + echo "" + + # give some debug output also + $P/.github/workflows/scripts/qemu-7-prepare.sh doit + + # if build was okay: + if [ -d vm1 ]; then + # Did we have a test failure? + if test -d vm1 && grep -vq 0 vm*/exitcode.txt ; then + echo "One or more tests failed:" + cat summary-failure-logs.txt + # On test failures, the github test results page will autoscroll to the + # bottom of summary-with-logs.txt. The first thing we want to see + # is the summary page of failures so print another copy of it at + # the bottom of the "page" for easy access. + echo "" + cat summary.txt + echo "" + echo "Full logs for download:" + echo '${{ steps.artifact-upload.outputs.artifact-url }}' + false + else + true + fi + fi + + cleanup: + if: always() + name: Cleanup + runs-on: ubuntu-latest + needs: [ qemu-vm ] + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/download-artifact@v4 + - name: Generating summary + run: .github/workflows/scripts/qemu-8-summary.sh + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 2 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 3 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 4 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 5 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 6 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 7 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 8 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 9 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 10 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 11 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 12 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 13 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 14 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 15 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 16 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 17 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 18 + - name: Generating summary... + run: .github/workflows/scripts/qemu-8-summary.sh 19 + - uses: actions/upload-artifact@v4 + with: + name: Summary Files + path: out-* diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index c25903ea1bee..957e674be2fa 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # shellcheck disable=SC2154 # # CDDL HEADER START @@ -208,6 +208,46 @@ find_runfile() { fi } +# Given a TAGS with a format like "1/3" or "2/3" then divide up the test list +# into portions and print that portion. So "1/3" for "the first third of the +# test tags". +# +# +split_tags() { + # Get numerator and denominator + NUM=$(echo $TAGS | cut -d/ -f1) + DEN=$(echo $TAGS | cut -d/ -f2) + # At the point this is called, RUNFILES will contain a comma separated + # list of full paths to the runfiles, like: + # + # "/home/hutter/qemu/tests/runfiles/common.run,/home/hutter/qemu/tests/runfiles/linux.run" + # + # So to get tags for our selected tests we do: + # + # 1. Remove unneeded chars: [],\ + # 2. Print out the last field of each tag line. This will be the tag + # for the test (like 'zpool_add'). + # 3. Remove duplicates between the runfiles. If the same tag is defined + # in multiple runfiles, then when you do '-T ' ZTS is smart + # enough to know to run the tag in each runfile. So '-T zpool_add' + # will run the zpool_add from common.run and linux.run. + # 4. Ignore the 'functional' tag since we only want individual tests + # 5. Print out the tests in our faction of all tests. This uses modulus + # so "1/3" will run tests 1,3,6,9 etc. That way the tests are + # interleaved so, say, "3/4" isn't running all the zpool_* tests that + # appear alphabetically at the end. + # 6. Remove trailing comma from list + # + # TAGS will then look like: + # + # "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..." + + cat ${RUNFILES/,/ } | tr -d [],\' | awk '/tags = /{print $NF}' | sort | \ + uniq | grep -v functional | \ + awk -v num=$NUM -v den=$DEN '{ if(NR % den == (num - 1)) {printf "%s,",$0}}' | \ + sed -E 's/,$//' +} + # # Symlink file if it appears under any of the given paths. # @@ -331,6 +371,10 @@ OPTIONS: -t PATH|NAME Run single test at PATH relative to test suite, or search for test by NAME -T TAGS Comma separated list of tags (default: 'functional') + Alternately, specify a fraction like "1/3" or "2/3" to + run the first third of tests or 2nd third of the tests. This + is useful for splitting up the test amongst different + runners. -u USER Run single test as USER (default: root) EXAMPLES: @@ -489,6 +533,8 @@ fi # TAGS=${TAGS:='functional'} + + # # Attempt to locate the runfiles describing the test workload. # @@ -509,6 +555,23 @@ done unset IFS RUNFILES=${R#,} +# The tag can be a fraction to indicate which portion of ZTS to run, Like +# +# "1/3": Run first one third of all tests in runfiles +# "2/3": Run second one third of all test in runfiles +# "6/10": Run 6th tenth of all tests in runfiles +# +# This is useful for splitting up the test across multiple runners. +# +# After this code block, TAGS will be transformed from something like +# "1/3" to a comma separate taglist, like: +# +# "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..." +# +if echo $TAGS | grep -Eq '^[0-9]+/[0-9]+$' ; then + TAGS=$(split_tags) +fi + # # This script should not be run as root. Instead the test user, which may # be a normal user account, needs to be configured such that it can