From 38b00b7a976c53d7ade33a8a429de031e718775d Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 29 Sep 2023 21:27:44 +0000 Subject: [PATCH 01/15] initial commit --- lib/base.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/base.sh b/lib/base.sh index c84330098..231da3cba 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -13,7 +13,7 @@ ch_lib=${ch_bin}/../lib verbose=0 DEBUG () { - if [ "$verbose" -ge 2 ]; then + if [ "$verbose" -ge 2 ] && [ -z "$quiet" ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -21,21 +21,25 @@ DEBUG () { } FATAL () { - printf 'error: ' 1>&2 - # shellcheck disable=SC2059 - printf "$@" 1>&2 - printf '\n' 1>&2 + if [ -z "$quiet" ]; then + printf 'error: ' 1>&2 + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 + fi exit 1 } INFO () { - # shellcheck disable=SC2059 - printf "$@" 1>&2 - printf '\n' 1>&2 + if [ -z "$quiet" ]; then + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 + fi } VERBOSE () { - if [ "$verbose" -ge 1 ]; then + if [ "$verbose" -ge 1 ] && [ -z "$quiet" ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -60,6 +64,10 @@ parse_basic_arg () { --help) usage 0 # exits ;; + -q|--quiet) + quiet="yes" + return 0 + ;; -v|--verbose) verbose=$((verbose+1)) return 0 From e8d4320d95ad7dfd225b1619ebeb0be3e6d8cfa7 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 2 Oct 2023 21:59:57 +0000 Subject: [PATCH 02/15] re-add errors, add test, document it --- doc/ch-convert.rst | 3 +++ lib/base.sh | 10 ++++------ test/run/ch-convert.bats | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/doc/ch-convert.rst b/doc/ch-convert.rst index 908ad2526..da588c5ba 100644 --- a/doc/ch-convert.rst +++ b/doc/ch-convert.rst @@ -55,6 +55,9 @@ producing the final format actually needed. :code:`-o`, :code:`--out-fmt FMT` Output image format is :code:`FMT`; inferred if omitted. + :code:`-q`, :code:`--quiet` + Suppress program output. Note that this does not suppress errors. + :code:`-s`, :code:`--storage DIR` Set the storage directory. Equivalent to the same option for :code:`ch-image(1)` and :code:`ch-run(1)`. diff --git a/lib/base.sh b/lib/base.sh index 231da3cba..8bd0e55f0 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -21,12 +21,10 @@ DEBUG () { } FATAL () { - if [ -z "$quiet" ]; then - printf 'error: ' 1>&2 - # shellcheck disable=SC2059 - printf "$@" 1>&2 - printf '\n' 1>&2 - fi + printf 'error: ' 1>&2 + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 exit 1 } diff --git a/test/run/ch-convert.bats b/test/run/ch-convert.bats index 9a4a2f61b..5b226239e 100644 --- a/test/run/ch-convert.bats +++ b/test/run/ch-convert.bats @@ -565,6 +565,23 @@ EOF } +@test 'ch-convert: --quiet' { + printf 'FROM alpine:3.17\n' | ch-image build -t tmpimg -f - "$BATS_TMPDIR" + + # successful convert + run ch-convert -q -i ch-image -o dir tmpimg "${BATS_TMPDIR}/tmpimg" + echo "$output" + [[ $status -eq 0 ]] + [[ -z "$output" ]] + + # failed convert + run ch-convert -q -i dir -o dir "${BATS_TMPDIR}/tmpimg" "${BATS_TMPDIR}/tmpimg2" + echo "$output" + [[ $status -eq 1 ]] + [[ "$output" = *'error: input and output formats must be different'* ]] +} + + @test 'ch-convert: dir -> ch-image -> X' { test_from ch-image } From f5edd5436ed2ec12ad7199d7e001b78dcfb12fa9 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 17 Oct 2023 21:12:49 +0000 Subject: [PATCH 03/15] add levels to quiet --- lib/base.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/base.sh b/lib/base.sh index 8bd0e55f0..5329fc321 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -12,6 +12,9 @@ ch_lib=${ch_bin}/../lib # Verbosity level; works the same as the Python code. verbose=0 +# Quiet level +quiet=0 + DEBUG () { if [ "$verbose" -ge 2 ] && [ -z "$quiet" ]; then # shellcheck disable=SC2059 @@ -29,7 +32,7 @@ FATAL () { } INFO () { - if [ -z "$quiet" ]; then + if [ "$quiet" -eq 0 ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -37,7 +40,7 @@ INFO () { } VERBOSE () { - if [ "$verbose" -ge 1 ] && [ -z "$quiet" ]; then + if [ "$verbose" -ge 1 ] && [ "$quiet" -le 0 ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -63,7 +66,7 @@ parse_basic_arg () { usage 0 # exits ;; -q|--quiet) - quiet="yes" + quiet=$((quiet+1)) return 0 ;; -v|--verbose) @@ -144,7 +147,7 @@ vset () { if [ -z "$value" ]; then value=no fi - if [ -z "$quiet" ]; then + if [ "$quiet" -eq 0 ]; then var_desc="$var_desc:" printf "%-*s %s (%s)\n" "$desc_width" "$var_desc" "$value" "$method" fi From e1c2eafc9f2796e90d8f1f218607d007ce0d0d49 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 18 Oct 2023 19:02:19 +0000 Subject: [PATCH 04/15] uhh it doesn't work --- bin/ch-convert | 6 +++--- lib/base.sh | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index 5399c2efb..9ad0ac57d 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -113,7 +113,7 @@ cv_dir_squash () { # Exclude build cache metadata. 64kiB block size based on Shane's # experiments. # shellcheck disable=SC2086 - mksquashfs "$1" "$2" $squash_xattr_arg -b 65536 -noappend -all-root -pf "$pflist" \ + quiet_process mksquashfs "$1" "$2" $squash_xattr_arg -b 65536 -noappend -all-root -pf "$pflist" \ -e "$1"/.git -e "$1"/.gitignore -e "$1"/ch/git.pickle # Zero the archive's internal modification time at bytes 8–11, 0-indexed # [1]. Newer SquashFS-Tools ≥4.3 have option “-fstime 0” to do this, but @@ -316,8 +316,8 @@ cv_tar_dir () { ex3="./${ex1}" VERBOSE "exclude patterns: ${ex1} ${ex2} ${ex3}" #shellcheck disable=SC2094,SC2086 - pv_ -s "$(stat -c%s "$1")" < "$1" \ - | tar x"$(tar_decompress_arg "$1")" -pC "$2" -f - \ + quiet_process pv_ -s "$(stat -c%s "$1")" < "$1" \ + | quiet_process tar x"$(tar_decompress_arg "$1")" -pC "$2" -f - \ --xform 's|^\./||x' --strip-components=$strip_ct \ --anchored --no-wildcards-match-slash $tar_xattr_args \ --exclude="$ex1" --exclude="$ex2" --exclude="$ex3" diff --git a/lib/base.sh b/lib/base.sh index 5329fc321..977fa797a 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -89,6 +89,20 @@ parse_basic_args () { done } +# Redirect standard streams (or not) depending on “quiet” level. See table in +# FAQ. +quiet_process () { + printf "quiet called\n" + if [ $quiet -ge 1 ]; then + printf "shh I'm being quiet\n" + $@ 1>/dev/null + fi + if [ $quiet -ge 3 ]; then + printf "I'm being double-quiet\n" + $@ 2>/dev/null + fi +} + # Convert container registry path to filesystem compatible path. # # NOTE: This is used both to name user-visible stuff like tarballs as well as From a5d5635508aae662b30a66327ef3d4bd0842d93d Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 24 Oct 2023 16:54:14 +0000 Subject: [PATCH 05/15] change approach for pv_ [skip ci] --- bin/ch-convert | 2 +- lib/base.sh | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index 9ad0ac57d..9416d9ef8 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -316,7 +316,7 @@ cv_tar_dir () { ex3="./${ex1}" VERBOSE "exclude patterns: ${ex1} ${ex2} ${ex3}" #shellcheck disable=SC2094,SC2086 - quiet_process pv_ -s "$(stat -c%s "$1")" < "$1" \ + pv_ -s "$(stat -c%s "$1")" < "$1" \ | quiet_process tar x"$(tar_decompress_arg "$1")" -pC "$2" -f - \ --xform 's|^\./||x' --strip-components=$strip_ct \ --anchored --no-wildcards-match-slash $tar_xattr_args \ diff --git a/lib/base.sh b/lib/base.sh index 977fa797a..e409ea30f 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -92,13 +92,10 @@ parse_basic_args () { # Redirect standard streams (or not) depending on “quiet” level. See table in # FAQ. quiet_process () { - printf "quiet called\n" if [ $quiet -ge 1 ]; then - printf "shh I'm being quiet\n" $@ 1>/dev/null fi if [ $quiet -ge 3 ]; then - printf "I'm being double-quiet\n" $@ 2>/dev/null fi } @@ -208,7 +205,11 @@ fi # WARNING: You must pipe in the file because arguments are ignored if this is # cat(1). (We also don't want a progress bar if stdin is not a terminal, but # pv takes care of that.) -if command -v pv > /dev/null 2>&1; then +if [ "$quiet" -ge 1 ]; then + pv_ () { + : # noop if “--quiet” + } +elif command -v pv > /dev/null 2>&1; then pv_ () { pv -pteb "$@" } From 9b33ffec2b4bbdd15a301c33fdfb337da8045903 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 24 Oct 2023 20:24:54 +0000 Subject: [PATCH 06/15] silence dd, update quiet_process --- bin/ch-convert | 2 +- lib/base.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index 9416d9ef8..9eca0d641 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -118,7 +118,7 @@ cv_dir_squash () { # Zero the archive's internal modification time at bytes 8–11, 0-indexed # [1]. Newer SquashFS-Tools ≥4.3 have option “-fstime 0” to do this, but # CentOS 7 comes with 4.2. [1]: https://dr-emann.github.io/squashfs/ - printf '\x00\x00\x00\x00' | dd of="$2" bs=1 count=4 seek=8 conv=notrunc + printf '\x00\x00\x00\x00' | quiet_process dd of="$2" bs=1 count=4 seek=8 conv=notrunc status=none rm "$pflist" } diff --git a/lib/base.sh b/lib/base.sh index e409ea30f..373cd1019 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -95,7 +95,7 @@ quiet_process () { if [ $quiet -ge 1 ]; then $@ 1>/dev/null fi - if [ $quiet -ge 3 ]; then + if [ $quiet -ge 2 ]; then $@ 2>/dev/null fi } From e3b7c6cd3807f1bfa5e8a5b769bed9970f3d21b3 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 26 Oct 2023 21:38:15 +0000 Subject: [PATCH 07/15] fix it? --- lib/base.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/base.sh b/lib/base.sh index 373cd1019..b68f6741a 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -93,10 +93,12 @@ parse_basic_args () { # FAQ. quiet_process () { if [ $quiet -ge 1 ]; then - $@ 1>/dev/null - fi - if [ $quiet -ge 2 ]; then - $@ 2>/dev/null + "$@" 1>/dev/null + if [ $quiet -ge 2 ]; then + "$@" 2>/dev/null + fi + else + "$@" fi } @@ -205,11 +207,7 @@ fi # WARNING: You must pipe in the file because arguments are ignored if this is # cat(1). (We also don't want a progress bar if stdin is not a terminal, but # pv takes care of that.) -if [ "$quiet" -ge 1 ]; then - pv_ () { - : # noop if “--quiet” - } -elif command -v pv > /dev/null 2>&1; then +if command -v pv > /dev/null 2>&1 && [ "$quiet" -lt 1 ]; then pv_ () { pv -pteb "$@" } From 33da19b1d3f57819b8a95ca921676560197b8d81 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Thu, 26 Oct 2023 22:22:19 +0000 Subject: [PATCH 08/15] quiet everything --- bin/ch-convert | 76 +++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index 9eca0d641..428a500d4 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -51,7 +51,7 @@ cv_chimage_dir () { # completely half-assed performance test suggests that's about the same. dir_make "$2" # shellcheck disable=SC2086 - tar_ "$img" | tar xf - $tar_xattr_args -pC "$2" + tar_ "$img" | quiet_process tar xf - $tar_xattr_args -pC "$2" dir_fixup "$2" } @@ -79,14 +79,14 @@ cv_chimage_tar () { tar_out_validate "$2" img=$(chimage_path "$1") INFO 'exporting ...' - tar_ "$img" | gzip_ > "$2" + tar_ "$img" | quiet_process gzip_ > "$2" } cv_dir_chimage () { dir_in_validate "$1" chimage_out_validate "$2" INFO 'importing ...' - ch-image import "$1" "$2" # FIXME: no progress meter + quiet_process ch-image import "$1" "$2" # FIXME: no progress meter } cv_dir_docker () { @@ -108,7 +108,7 @@ cv_dir_squash () { squash_out_validate "$2" pflist=${tmpdir}/pseudofiles INFO 'packing ...' - touch "$pflist" + quiet_process touch "$pflist" mount_points_ensure "$1" "$pflist" # Exclude build cache metadata. 64kiB block size based on Shane's # experiments. @@ -119,7 +119,7 @@ cv_dir_squash () { # [1]. Newer SquashFS-Tools ≥4.3 have option “-fstime 0” to do this, but # CentOS 7 comes with 4.2. [1]: https://dr-emann.github.io/squashfs/ printf '\x00\x00\x00\x00' | quiet_process dd of="$2" bs=1 count=4 seek=8 conv=notrunc status=none - rm "$pflist" + quiet_process rm "$pflist" } cv_dir_tar () { @@ -128,7 +128,7 @@ cv_dir_tar () { # Don't add essential files & directories because that will happen later # when converted to dir or squash. INFO 'packing ...' - tar_ "$1" | gzip_ > "$2" + tar_ "$1" | quiet_process gzip_ > "$2" } cv_docker_chimage () { @@ -149,7 +149,7 @@ cv_docker_podman () { docker_out=${tmpdir}/weirdal.tar.gz cv_docker_tar "$1" "$docker_out" # FIXME: needlessly compresses cv_tar_podman "$docker_out" "$2" - rm "$docker_out" + quiet_process rm "$docker_out" } cv_docker_squash () { @@ -182,7 +182,7 @@ cv_podman_docker () { podman_out=${tmpdir}/weirdal.tar.gz cv_podman_tar "$1" "$podman_out" # FIXME: needlesly compresses cv_tar_docker "$podman_out" "$2" - rm "$podman_out" + quiet_process rm "$podman_out" } cv_podman_squash () { @@ -203,7 +203,7 @@ cv_squash_chimage () { unsquash_dir=${tmpdir}/weirdal cv_squash_dir "$1" "$unsquash_dir" cv_dir_chimage "$unsquash_dir" "$2" - rm -Rf --one-file-system "$unsquash_dir" + quiet_process rm -Rf --one-file-system "$unsquash_dir" } cv_squash_dir () { @@ -246,7 +246,7 @@ cv_squash_dir () { # and confirm that it’s successful. # # (https://unix.stackexchange.com/a/583819) - unsquashfs -f -d "$2" -user-xattrs "$1" + quiet_process unsquashfs -f -d "$2" -user-xattrs "$1" umask "$umask_" dir_fixup "$2" } @@ -269,14 +269,14 @@ cv_squash_tar () { unsquash_dir=${tmpdir}/weirdal cv_squash_dir "$1" "$unsquash_dir" cv_dir_tar "$unsquash_dir" "$2" - rm -Rf --one-file-system "$unsquash_dir" + quiet_process rm -Rf --one-file-system "$unsquash_dir" } cv_tar_chimage () { tar_in_validate "$1" chimage_out_validate "$2" INFO 'importing ...' - ch-image import "$1" "$2" # FIXME: no progress meter + quiet_process ch-image import "$1" "$2" # FIXME: no progress meter } cv_tar_dir () { @@ -342,7 +342,7 @@ cv_tar_squash () { tar_dir=${tmpdir}/weirdal cv_tar_dir "$1" "$tar_dir" cv_dir_squash "$tar_dir" "$2" - rm -Rf --one-file-system "$tar_dir" + quiet_process rm -Rf --one-file-system "$tar_dir" } ## Dockman functions ## @@ -357,7 +357,7 @@ chimage_to_dockman () { chimage_tar=${tmpdir}/weirdal.tar.gz cv_chimage_tar "$2" "$chimage_tar" # FIXME: needlessly compresses? tar_to_dockman "$1" "$chimage_tar" "$3" - rm "$chimage_tar" + quiet_process rm "$chimage_tar" } dir_to_dockman () { @@ -367,7 +367,7 @@ dir_to_dockman () { # directory anyway to send it to the Docker daemon. cv_dir_tar "$2" "$dirtar" # FIXME: needlessly compresses tar_to_dockman "$1" "$dirtar" "$3" - rm "$dirtar" + quiet_process rm "$dirtar" } dm_fmt_name () { @@ -394,21 +394,21 @@ dockman_to_chimage () { dockman_out=${tmpdir}/weirdal.tar.gz "cv_${1}tar" "$2" "$dockman_out" # FIXME: needlessly compresses cv_tar_chimage "$dockman_out" "$3" - rm "$dockman_out" + quiet_process rm "$dockman_out" } dockman_to_dir () { dockman_out=${tmpdir}/weirdal.tar.gz "dockman_to_tar" "$1" "$2" "$dockman_out" # FIXME: needlessly compresses cv_tar_dir "$dockman_out" "$3" - rm "$dockman_out" + quiet_process rm "$dockman_out" } dockman_to_squash () { dockman_dir=${tmpdir}/weirdal dockman_to_dir "$1" "$2" "$dockman_dir" # FIXME: needlessly compresses cv_dir_squash "$dockman_dir" "$3" - rm -Rf --one-file-system "$dockman_dir" + quiet_process rm -Rf --one-file-system "$dockman_dir" } dockman_to_tar () { @@ -417,7 +417,7 @@ dockman_to_tar () { INFO 'exporting ...' cid=$("$1" create --read-only "$2" /bin/true) # cmd needed but not run size=$("$1" image inspect "$2" --format='{{.Size}}') - "$1" export "$cid" | pv_ -s "$size" > "$tmptar" + quiet_process "$1" export "$cid" | pv_ -s "$size" > "$tmptar" "$1" rm "$cid" > /dev/null INFO 'adding environment ...' "$1" inspect "$2" \ @@ -425,12 +425,12 @@ dockman_to_tar () { # The tar flavor Docker gives us does not support UIDs or GIDs greater # than 2**21, so use 0/0 rather than what’s on the filesystem. See #1573. # shellcheck disable=SC2086 - tar rf "$tmptar" -b1 -P --owner=0 --group=0 $tar_xattr_args \ + quiet_process tar rf "$tmptar" -b1 -P --owner=0 --group=0 $tar_xattr_args \ --xform="s|${tmpenv}|ch/environment|" "$tmpenv" INFO 'compressing ...' - pv_ < "$tmptar" | gzip_ -6 > "$3" - rm "$tmptar" - rm "$tmpenv" + pv_ < "$tmptar" | quiet_process gzip_ -6 > "$3" + quiet_process rm "$tmptar" + quiet_process rm "$tmpenv" } squash_to_dockman () { @@ -438,20 +438,20 @@ squash_to_dockman () { cv_squash_tar "$2" "$unsquash_tar" dockman=$(echo "$1" | tr -d "_") # remove trailing underscore "cv_tar_$dockman" "$unsquash_tar" "$3" - rm "$unsquash_tar" + quiet_process rm "$unsquash_tar" } tar_to_dockman () { INFO "importing ..." tmpimg=$(mktemp -u tmpimg.XXXXXX | tr '[:upper:]' '[:lower:]') - "$1" import "$2" "$tmpimg" # FIXME: no progress meter + quiet_process "$1" import "$2" "$tmpimg" # FIXME: no progress meter # Podman imports our tarballs with rw------- permissions on “/ch” (i.e., # no execute), which causes all kinds of breakage. Work around that. - "$1" build -t "$3" - <> "$2" + quiet_process echo "${i} d 755 root root" >> "$2" else - mkdir "${1}/${i}" + quiet_process mkdir "${1}/${i}" fi fi done @@ -686,9 +686,9 @@ mount_points_ensure () { for i in etc/hosts etc/resolv.conf; do if ! exist_p "${1}/${i}"; then if [ -n "$2" ]; then - echo "${i} f 644 root root true" >> "$2" + quiet_process echo "${i} f 644 root root true" >> "$2" else - touch "${1}/${i}" + quiet_process touch "${1}/${i}" fi fi done @@ -711,7 +711,7 @@ path_noclobber () { # Produce a tarbomb because Docker requires tarbombs. tar_ () { # shellcheck disable=SC2086 - ( cd "$1" && tar cf - $tar_xattr_args \ + ( quiet_process cd "$1" && quiet_process tar cf - $tar_xattr_args \ --exclude=./.git \ --exclude=./.gitignore \ --exclude=./ch/git.pickle . ) | pv_ @@ -915,6 +915,6 @@ if [ -z "$dry_run" ]; then "$in_desc" "$out_desc" fi -rmdir "$tmpdir" +quiet_process rmdir "$tmpdir" INFO 'done' From 5e993805f71b3cd7b8617a4666ba189f3ea4174c Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Fri, 27 Oct 2023 21:59:30 +0000 Subject: [PATCH 09/15] oops --- bin/ch-convert | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index 428a500d4..e1115dca4 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -549,9 +549,9 @@ tar_out_validate () { # Return the path to image $1 in ch-image storage. chimage_path () { if [ -z "$cli_storage" ]; then - quiet_process echo "$(ch-image gestalt storage-path)/img/$(tag_to_path "$1")" + echo "$(ch-image gestalt storage-path)/img/$(tag_to_path "$1")" else - quiet_process echo "$cli_storage/img/$(tag_to_path "$1")" + echo "$cli_storage/img/$(tag_to_path "$1")" fi } @@ -711,7 +711,7 @@ path_noclobber () { # Produce a tarbomb because Docker requires tarbombs. tar_ () { # shellcheck disable=SC2086 - ( quiet_process cd "$1" && quiet_process tar cf - $tar_xattr_args \ + ( cd "$1" && tar cf - $tar_xattr_args \ --exclude=./.git \ --exclude=./.gitignore \ --exclude=./ch/git.pickle . ) | pv_ From 47127bc420a10a0fda1018f2dc6b3574b16b2813 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Mon, 30 Oct 2023 17:05:14 +0000 Subject: [PATCH 10/15] screaming crying throwing up --- bin/ch-convert | 2 +- lib/base.sh | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index e1115dca4..a94c0dc32 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -109,7 +109,7 @@ cv_dir_squash () { pflist=${tmpdir}/pseudofiles INFO 'packing ...' quiet_process touch "$pflist" - mount_points_ensure "$1" "$pflist" + quiet_process mount_points_ensure "$1" "$pflist" # Exclude build cache metadata. 64kiB block size based on Shane's # experiments. # shellcheck disable=SC2086 diff --git a/lib/base.sh b/lib/base.sh index b68f6741a..059ee104f 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -93,9 +93,10 @@ parse_basic_args () { # FAQ. quiet_process () { if [ $quiet -ge 1 ]; then - "$@" 1>/dev/null if [ $quiet -ge 2 ]; then - "$@" 2>/dev/null + "$@" 1>/dev/null 2>/dev/null + else + "$@" 1>/dev/null fi else "$@" @@ -203,17 +204,16 @@ else } fi -# Use pv(1) to show a progress bar, if it's available, otherwise cat(1). -# WARNING: You must pipe in the file because arguments are ignored if this is -# cat(1). (We also don't want a progress bar if stdin is not a terminal, but -# pv takes care of that.) -if command -v pv > /dev/null 2>&1 && [ "$quiet" -lt 1 ]; then - pv_ () { +# Use pv(1) to show a progress bar, if it’s available and the quiet level is +# less than one, otherwise cat(1). WARNING: You must pipe in the file because +# arguments are ignored if this is cat(1). (We also don’t want a progress bar if +# stdin is not a terminal, but pv takes care of that). Note that we put the if +# statement in the scope of the function because doing so ensures that it gets +# evaulated after “quiet” is assigned an appropriate value by “parse_basic_arg”. +pv_ () { + if command -v pv > /dev/null 2>&1 && [ "$quiet" -lt 1 ]; then pv -pteb "$@" - } -else - pv_ () { - # Arguments may be present, but we ignore them. + else cat - } -fi + fi +} From 6edb69a0e1d291f0252292839f20ac1cc9a3a87c Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 31 Oct 2023 15:27:13 +0000 Subject: [PATCH 11/15] make it work (?) --- bin/ch-convert | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index a94c0dc32..078b70ccb 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -79,7 +79,7 @@ cv_chimage_tar () { tar_out_validate "$2" img=$(chimage_path "$1") INFO 'exporting ...' - tar_ "$img" | quiet_process gzip_ > "$2" + tar_ "$img" | gzip_ > "$2" } cv_dir_chimage () { @@ -128,7 +128,7 @@ cv_dir_tar () { # Don't add essential files & directories because that will happen later # when converted to dir or squash. INFO 'packing ...' - tar_ "$1" | quiet_process gzip_ > "$2" + tar_ "$1" | gzip_ > "$2" } cv_docker_chimage () { @@ -428,7 +428,7 @@ dockman_to_tar () { quiet_process tar rf "$tmptar" -b1 -P --owner=0 --group=0 $tar_xattr_args \ --xform="s|${tmpenv}|ch/environment|" "$tmpenv" INFO 'compressing ...' - pv_ < "$tmptar" | quiet_process gzip_ -6 > "$3" + pv_ < "$tmptar" | gzip_ -6 > "$3" quiet_process rm "$tmptar" quiet_process rm "$tmpenv" } From 6316f58e20cf20a415d2e082aadd326839894094 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 31 Oct 2023 15:37:13 +0000 Subject: [PATCH 12/15] make code less terrible --- lib/base.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/base.sh b/lib/base.sh index 059ee104f..c48e7743b 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -92,12 +92,10 @@ parse_basic_args () { # Redirect standard streams (or not) depending on “quiet” level. See table in # FAQ. quiet_process () { - if [ $quiet -ge 1 ]; then - if [ $quiet -ge 2 ]; then - "$@" 1>/dev/null 2>/dev/null - else - "$@" 1>/dev/null - fi + if [ $quiet -ge 2 ]; then + "$@" 1>/dev/null 2>/dev/null + elif [ $quiet -ge 1 ]; then + "$@" 1>/dev/null else "$@" fi From e509e29118d9308b643477311e1d88579a062882 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Tue, 31 Oct 2023 15:58:42 +0000 Subject: [PATCH 13/15] document it, add error, remove test --- bin/ch-convert | 3 +++ doc/ch-convert.rst | 3 ++- test/run/ch-convert.bats | 18 ------------------ 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index 078b70ccb..b0287e484 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -865,6 +865,9 @@ done if [ "$#" -ne 2 ]; then usage fi +if [ $verbose -gt 0 ] && [ $quiet -gt 0 ]; then + FATAL "incompatible options: --quiet, --verbose" +fi if [ -n "$no_xattrs" ]; then tar_xattr_args= squash_xattr_arg= diff --git a/doc/ch-convert.rst b/doc/ch-convert.rst index da588c5ba..3286e8f46 100644 --- a/doc/ch-convert.rst +++ b/doc/ch-convert.rst @@ -56,7 +56,8 @@ producing the final format actually needed. Output image format is :code:`FMT`; inferred if omitted. :code:`-q`, :code:`--quiet` - Suppress program output. Note that this does not suppress errors. + Be quieter; can be repeated. Incompatible with :code:`-v`. See the :ref:`FAQ + entry on verbosity ` for details. :code:`-s`, :code:`--storage DIR` Set the storage directory. Equivalent to the same option for :code:`ch-image(1)` diff --git a/test/run/ch-convert.bats b/test/run/ch-convert.bats index 5b226239e..e8506fa9f 100644 --- a/test/run/ch-convert.bats +++ b/test/run/ch-convert.bats @@ -564,24 +564,6 @@ EOF [[ $output = *"user:$USER:r--"* ]] } - -@test 'ch-convert: --quiet' { - printf 'FROM alpine:3.17\n' | ch-image build -t tmpimg -f - "$BATS_TMPDIR" - - # successful convert - run ch-convert -q -i ch-image -o dir tmpimg "${BATS_TMPDIR}/tmpimg" - echo "$output" - [[ $status -eq 0 ]] - [[ -z "$output" ]] - - # failed convert - run ch-convert -q -i dir -o dir "${BATS_TMPDIR}/tmpimg" "${BATS_TMPDIR}/tmpimg2" - echo "$output" - [[ $status -eq 1 ]] - [[ "$output" = *'error: input and output formats must be different'* ]] -} - - @test 'ch-convert: dir -> ch-image -> X' { test_from ch-image } From 0ed7250fa91a456428f26f3c67f6072b6936a217 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 1 Nov 2023 15:47:30 +0000 Subject: [PATCH 14/15] rename function [skip ci] --- bin/ch-convert | 72 +++++++++++++++++++++++++------------------------- lib/base.sh | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index b0287e484..b5ebaa4c0 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -51,7 +51,7 @@ cv_chimage_dir () { # completely half-assed performance test suggests that's about the same. dir_make "$2" # shellcheck disable=SC2086 - tar_ "$img" | quiet_process tar xf - $tar_xattr_args -pC "$2" + tar_ "$img" | quiet tar xf - $tar_xattr_args -pC "$2" dir_fixup "$2" } @@ -86,7 +86,7 @@ cv_dir_chimage () { dir_in_validate "$1" chimage_out_validate "$2" INFO 'importing ...' - quiet_process ch-image import "$1" "$2" # FIXME: no progress meter + quiet ch-image import "$1" "$2" # FIXME: no progress meter } cv_dir_docker () { @@ -108,18 +108,18 @@ cv_dir_squash () { squash_out_validate "$2" pflist=${tmpdir}/pseudofiles INFO 'packing ...' - quiet_process touch "$pflist" - quiet_process mount_points_ensure "$1" "$pflist" + quiet touch "$pflist" + quiet mount_points_ensure "$1" "$pflist" # Exclude build cache metadata. 64kiB block size based on Shane's # experiments. # shellcheck disable=SC2086 - quiet_process mksquashfs "$1" "$2" $squash_xattr_arg -b 65536 -noappend -all-root -pf "$pflist" \ + quiet mksquashfs "$1" "$2" $squash_xattr_arg -b 65536 -noappend -all-root -pf "$pflist" \ -e "$1"/.git -e "$1"/.gitignore -e "$1"/ch/git.pickle # Zero the archive's internal modification time at bytes 8–11, 0-indexed # [1]. Newer SquashFS-Tools ≥4.3 have option “-fstime 0” to do this, but # CentOS 7 comes with 4.2. [1]: https://dr-emann.github.io/squashfs/ - printf '\x00\x00\x00\x00' | quiet_process dd of="$2" bs=1 count=4 seek=8 conv=notrunc status=none - quiet_process rm "$pflist" + printf '\x00\x00\x00\x00' | quiet dd of="$2" bs=1 count=4 seek=8 conv=notrunc status=none + quiet rm "$pflist" } cv_dir_tar () { @@ -149,7 +149,7 @@ cv_docker_podman () { docker_out=${tmpdir}/weirdal.tar.gz cv_docker_tar "$1" "$docker_out" # FIXME: needlessly compresses cv_tar_podman "$docker_out" "$2" - quiet_process rm "$docker_out" + quiet rm "$docker_out" } cv_docker_squash () { @@ -182,7 +182,7 @@ cv_podman_docker () { podman_out=${tmpdir}/weirdal.tar.gz cv_podman_tar "$1" "$podman_out" # FIXME: needlesly compresses cv_tar_docker "$podman_out" "$2" - quiet_process rm "$podman_out" + quiet rm "$podman_out" } cv_podman_squash () { @@ -203,7 +203,7 @@ cv_squash_chimage () { unsquash_dir=${tmpdir}/weirdal cv_squash_dir "$1" "$unsquash_dir" cv_dir_chimage "$unsquash_dir" "$2" - quiet_process rm -Rf --one-file-system "$unsquash_dir" + quiet rm -Rf --one-file-system "$unsquash_dir" } cv_squash_dir () { @@ -246,7 +246,7 @@ cv_squash_dir () { # and confirm that it’s successful. # # (https://unix.stackexchange.com/a/583819) - quiet_process unsquashfs -f -d "$2" -user-xattrs "$1" + quiet unsquashfs -f -d "$2" -user-xattrs "$1" umask "$umask_" dir_fixup "$2" } @@ -269,14 +269,14 @@ cv_squash_tar () { unsquash_dir=${tmpdir}/weirdal cv_squash_dir "$1" "$unsquash_dir" cv_dir_tar "$unsquash_dir" "$2" - quiet_process rm -Rf --one-file-system "$unsquash_dir" + quiet rm -Rf --one-file-system "$unsquash_dir" } cv_tar_chimage () { tar_in_validate "$1" chimage_out_validate "$2" INFO 'importing ...' - quiet_process ch-image import "$1" "$2" # FIXME: no progress meter + quiet ch-image import "$1" "$2" # FIXME: no progress meter } cv_tar_dir () { @@ -317,7 +317,7 @@ cv_tar_dir () { VERBOSE "exclude patterns: ${ex1} ${ex2} ${ex3}" #shellcheck disable=SC2094,SC2086 pv_ -s "$(stat -c%s "$1")" < "$1" \ - | quiet_process tar x"$(tar_decompress_arg "$1")" -pC "$2" -f - \ + | quiet tar x"$(tar_decompress_arg "$1")" -pC "$2" -f - \ --xform 's|^\./||x' --strip-components=$strip_ct \ --anchored --no-wildcards-match-slash $tar_xattr_args \ --exclude="$ex1" --exclude="$ex2" --exclude="$ex3" @@ -342,7 +342,7 @@ cv_tar_squash () { tar_dir=${tmpdir}/weirdal cv_tar_dir "$1" "$tar_dir" cv_dir_squash "$tar_dir" "$2" - quiet_process rm -Rf --one-file-system "$tar_dir" + quiet rm -Rf --one-file-system "$tar_dir" } ## Dockman functions ## @@ -357,7 +357,7 @@ chimage_to_dockman () { chimage_tar=${tmpdir}/weirdal.tar.gz cv_chimage_tar "$2" "$chimage_tar" # FIXME: needlessly compresses? tar_to_dockman "$1" "$chimage_tar" "$3" - quiet_process rm "$chimage_tar" + quiet rm "$chimage_tar" } dir_to_dockman () { @@ -367,7 +367,7 @@ dir_to_dockman () { # directory anyway to send it to the Docker daemon. cv_dir_tar "$2" "$dirtar" # FIXME: needlessly compresses tar_to_dockman "$1" "$dirtar" "$3" - quiet_process rm "$dirtar" + quiet rm "$dirtar" } dm_fmt_name () { @@ -394,21 +394,21 @@ dockman_to_chimage () { dockman_out=${tmpdir}/weirdal.tar.gz "cv_${1}tar" "$2" "$dockman_out" # FIXME: needlessly compresses cv_tar_chimage "$dockman_out" "$3" - quiet_process rm "$dockman_out" + quiet rm "$dockman_out" } dockman_to_dir () { dockman_out=${tmpdir}/weirdal.tar.gz "dockman_to_tar" "$1" "$2" "$dockman_out" # FIXME: needlessly compresses cv_tar_dir "$dockman_out" "$3" - quiet_process rm "$dockman_out" + quiet rm "$dockman_out" } dockman_to_squash () { dockman_dir=${tmpdir}/weirdal dockman_to_dir "$1" "$2" "$dockman_dir" # FIXME: needlessly compresses cv_dir_squash "$dockman_dir" "$3" - quiet_process rm -Rf --one-file-system "$dockman_dir" + quiet rm -Rf --one-file-system "$dockman_dir" } dockman_to_tar () { @@ -417,7 +417,7 @@ dockman_to_tar () { INFO 'exporting ...' cid=$("$1" create --read-only "$2" /bin/true) # cmd needed but not run size=$("$1" image inspect "$2" --format='{{.Size}}') - quiet_process "$1" export "$cid" | pv_ -s "$size" > "$tmptar" + quiet "$1" export "$cid" | pv_ -s "$size" > "$tmptar" "$1" rm "$cid" > /dev/null INFO 'adding environment ...' "$1" inspect "$2" \ @@ -425,12 +425,12 @@ dockman_to_tar () { # The tar flavor Docker gives us does not support UIDs or GIDs greater # than 2**21, so use 0/0 rather than what’s on the filesystem. See #1573. # shellcheck disable=SC2086 - quiet_process tar rf "$tmptar" -b1 -P --owner=0 --group=0 $tar_xattr_args \ + quiet tar rf "$tmptar" -b1 -P --owner=0 --group=0 $tar_xattr_args \ --xform="s|${tmpenv}|ch/environment|" "$tmpenv" INFO 'compressing ...' pv_ < "$tmptar" | gzip_ -6 > "$3" - quiet_process rm "$tmptar" - quiet_process rm "$tmpenv" + quiet rm "$tmptar" + quiet rm "$tmpenv" } squash_to_dockman () { @@ -438,20 +438,20 @@ squash_to_dockman () { cv_squash_tar "$2" "$unsquash_tar" dockman=$(echo "$1" | tr -d "_") # remove trailing underscore "cv_tar_$dockman" "$unsquash_tar" "$3" - quiet_process rm "$unsquash_tar" + quiet rm "$unsquash_tar" } tar_to_dockman () { INFO "importing ..." tmpimg=$(mktemp -u tmpimg.XXXXXX | tr '[:upper:]' '[:lower:]') - quiet_process "$1" import "$2" "$tmpimg" # FIXME: no progress meter + quiet "$1" import "$2" "$tmpimg" # FIXME: no progress meter # Podman imports our tarballs with rw------- permissions on “/ch” (i.e., # no execute), which causes all kinds of breakage. Work around that. - quiet_process "$1" build -t "$3" - <> "$2" + quiet echo "${i} d 755 root root" >> "$2" else - quiet_process mkdir "${1}/${i}" + quiet mkdir "${1}/${i}" fi fi done @@ -686,9 +686,9 @@ mount_points_ensure () { for i in etc/hosts etc/resolv.conf; do if ! exist_p "${1}/${i}"; then if [ -n "$2" ]; then - quiet_process echo "${i} f 644 root root true" >> "$2" + quiet echo "${i} f 644 root root true" >> "$2" else - quiet_process touch "${1}/${i}" + quiet touch "${1}/${i}" fi fi done @@ -918,6 +918,6 @@ if [ -z "$dry_run" ]; then "$in_desc" "$out_desc" fi -quiet_process rmdir "$tmpdir" +quiet rmdir "$tmpdir" INFO 'done' diff --git a/lib/base.sh b/lib/base.sh index c48e7743b..331080474 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -91,7 +91,7 @@ parse_basic_args () { # Redirect standard streams (or not) depending on “quiet” level. See table in # FAQ. -quiet_process () { +quiet () { if [ $quiet -ge 2 ]; then "$@" 1>/dev/null 2>/dev/null elif [ $quiet -ge 1 ]; then From 69283f94a1187fe2367d3e141630e0a672050179 Mon Sep 17 00:00:00 2001 From: Lucas Caudill Date: Wed, 1 Nov 2023 16:33:47 +0000 Subject: [PATCH 15/15] suggestions --- bin/ch-convert | 15 ++++++++------- lib/base.sh | 28 ++++++++++++++++------------ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/bin/ch-convert b/bin/ch-convert index b5ebaa4c0..74bdb03c4 100755 --- a/bin/ch-convert +++ b/bin/ch-convert @@ -86,6 +86,9 @@ cv_dir_chimage () { dir_in_validate "$1" chimage_out_validate "$2" INFO 'importing ...' + # “ch-image” recognizes “-q” as an argument, but we choose to quiet it with + # “quiet” instead here because doing so is simpler from the perspective of + # “ch-convert”. quiet ch-image import "$1" "$2" # FIXME: no progress meter } @@ -109,12 +112,13 @@ cv_dir_squash () { pflist=${tmpdir}/pseudofiles INFO 'packing ...' quiet touch "$pflist" - quiet mount_points_ensure "$1" "$pflist" + mount_points_ensure "$1" "$pflist" # Exclude build cache metadata. 64kiB block size based on Shane's # experiments. # shellcheck disable=SC2086 - quiet mksquashfs "$1" "$2" $squash_xattr_arg -b 65536 -noappend -all-root -pf "$pflist" \ - -e "$1"/.git -e "$1"/.gitignore -e "$1"/ch/git.pickle + quiet mksquashfs "$1" "$2" $squash_xattr_arg -b 65536 -noappend -all-root \ + -pf "$pflist" -e "$1"/.git -e "$1"/.gitignore \ + -e "$1"/ch/git.pickle # Zero the archive's internal modification time at bytes 8–11, 0-indexed # [1]. Newer SquashFS-Tools ≥4.3 have option “-fstime 0” to do this, but # CentOS 7 comes with 4.2. [1]: https://dr-emann.github.io/squashfs/ @@ -865,9 +869,6 @@ done if [ "$#" -ne 2 ]; then usage fi -if [ $verbose -gt 0 ] && [ $quiet -gt 0 ]; then - FATAL "incompatible options: --quiet, --verbose" -fi if [ -n "$no_xattrs" ]; then tar_xattr_args= squash_xattr_arg= @@ -877,7 +878,7 @@ else fi in_desc=$1 out_desc=$2 -VERBOSE "verbose level: ${verbose}" +VERBOSE "verbose level: ${log_level}" if command -v ch-image > /dev/null 2>&1; then have_ch_image=yes diff --git a/lib/base.sh b/lib/base.sh index 331080474..8db0df61c 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -9,14 +9,12 @@ ch_lib=${ch_bin}/../lib . "${ch_lib}/version.sh" -# Verbosity level; works the same as the Python code. -verbose=0 - -# Quiet level -quiet=0 +# Log level. Incremented by “--verbose” and decremented by “--quiet”, as in the +# Python code. +log_level=0 DEBUG () { - if [ "$verbose" -ge 2 ] && [ -z "$quiet" ]; then + if [ "$log_level" -ge 2 ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -32,7 +30,7 @@ FATAL () { } INFO () { - if [ "$quiet" -eq 0 ]; then + if [ "$log_level" -ge 0 ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -40,7 +38,7 @@ INFO () { } VERBOSE () { - if [ "$verbose" -ge 1 ] && [ "$quiet" -le 0 ]; then + if [ "$log_level" -ge 1 ]; then # shellcheck disable=SC2059 printf "$@" 1>&2 printf '\n' 1>&2 @@ -66,11 +64,17 @@ parse_basic_arg () { usage 0 # exits ;; -q|--quiet) - quiet=$((quiet+1)) + if [ $log_level -gt 0 ]; then + FATAL "incompatible options: --quiet, --verbose" + fi + log_level=$((log_level-1)) return 0 ;; -v|--verbose) - verbose=$((verbose+1)) + if [ $log_level -lt 0 ]; then + FATAL "incompatible options: --quiet, --verbose" + fi + log_level=$((log_level+1)) return 0 ;; --version) @@ -92,9 +96,9 @@ parse_basic_args () { # Redirect standard streams (or not) depending on “quiet” level. See table in # FAQ. quiet () { - if [ $quiet -ge 2 ]; then + if [ $log_level -lt -2 ]; then "$@" 1>/dev/null 2>/dev/null - elif [ $quiet -ge 1 ]; then + elif [ $log_level -lt -1 ]; then "$@" 1>/dev/null else "$@"