From 0647f69e86eaa3bff6d9234451cdc0cd5c6030ab Mon Sep 17 00:00:00 2001 From: Georgios Liargkovas <56384743+gliargovas@users.noreply.github.com> Date: Sat, 13 Jan 2024 23:33:24 +0200 Subject: [PATCH] Implement Multi-Lower-Layer Overlay Support through Merging (#122) * feat: add -L flag to support one or more lower directories * docs: shell completion, manpage, and readme update * test: add test to test merging multiple lowerdirs, and also invoking -L multiple times Note that currently -L implies -n (will not commit changes), support for that is being tracked in #142 --------- Co-authored-by: gliargovas Co-authored-by: Ezri Zhu Reviewed-by: Michael Greenberg Reviewed-by: Konstantinos Kallas --- README.md | 16 ++++++++ completions/try.bash | 5 ++- docs/try.1.md | 13 ++++++- test/merge_multiple_dirs.sh | 71 +++++++++++++++++++++++++++++++++++ try | 74 ++++++++++++++++++++++++++----------- 5 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 test/merge_multiple_dirs.sh diff --git a/README.md b/README.md index 1f12239a..5700820a 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,22 @@ $ try commit rustup-sandbox You can also run `try explore` to open your current shell in try, or `/try explore /tmp/tmp.X6OQb5tJwr` to explore an existing sandbox. +To specify multiple lower directories for overlay (by merging them together), you can use the `-L` (implies `-n`) flag followed by a colon-separated list of directories. The directories on the left have higher precedence and can overwrite the directories on the right: + +```ShellSession +$ try -D /tmp/sandbox1 "echo 'File 1 Contents - sandbox1' > file1.txt" +$ try -D /tmp/sandbox2 "echo 'File 2 Contents - sandbox2' > file2.txt" +$ try -D /tmp/sandbox3 "echo 'File 2 Contents - sandbox3' > file2.txt" + +# Now use the -L flag to merge both sandbox directories together, with sandbox3 having precedence over sandbox2 +$ try -L "/tmp/sandbox3:/tmp/sandbox2:/tmp/sandbox1" "cat file1.txt file2.txt" +File 1 Contents - sandbox1 +File 2 Contents - sandbox3 +``` + +In this example, `try` will merge `/sandbox1`, `/sandbox2` and `/sandbox3` together before mounting the overlay. This way, you can combine the contents of multiple `try` sandboxes. + + ## Known Issues Any command that interacts with other users/groups will fail since only the current user's UID/GID are mapped. However, the [future diff --git a/completions/try.bash b/completions/try.bash index 603c8004..d2a987bc 100644 --- a/completions/try.bash +++ b/completions/try.bash @@ -17,13 +17,16 @@ _try() { case "${cmd}" in (try) - opts="-n -y -v -h -x -i -D -U summary commit explore" + opts="-n -y -v -h -x -i -D -U -L summary commit explore" if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 fi case "${prev}" in + (-L) + COMPREPLY=($(compgen -d "${cur}")) + return 0;; (-D) COMPREPLY=($(compgen -d "${cur}")) return 0;; diff --git a/docs/try.1.md b/docs/try.1.md index 3ce8084d..d3ee1ed6 100644 --- a/docs/try.1.md +++ b/docs/try.1.md @@ -6,7 +6,7 @@ try - run a command in an overlay # SYNOPSIS -| try [-ny] [-i PATTERN] [-D DIR] [-U PATH] CMD [ARG ...] +| try [-ny] [-i PATTERN] [-D DIR] [-U PATH] [-L LOWER_DIRS] CMD [ARG ...] | try summary [DIR] | try commit [DIR] | try explore @@ -57,6 +57,11 @@ While using *try* you can choose to commit the result to the filesystem or compl : Use the unionfs helper implementation defined in the *PATH* (e.g., mergerfs, unionfs-fuse) instead of the default. This option is recommended in case OverlayFS fails. +-L *LOWER_DIRS* + +: Specify a colon-separated list of directories to be used as lower directories for the overlay, formatted as "dir1:dir2:...:dirn" (implies -n). + + ## Subcommands try summary *DIR* @@ -128,6 +133,12 @@ Alternatively, you can specify your own overlay directory as follows (note that try -D try_dir gunzip file.txt.gz ``` +To use multiple lower directories for overlay (by merging them), you can use the `-L` flag followed by a colon-separated list of directories. The directories on the left have higher precedence and can overwrite the directories on the right: + +``` +try -L /lowerdir1:/lowerdir2:/lowerdir3 gunzip file.txt.gz +``` + You can inspect the changes made inside a given overlay directory: ``` diff --git a/test/merge_multiple_dirs.sh b/test/merge_multiple_dirs.sh new file mode 100644 index 00000000..79524413 --- /dev/null +++ b/test/merge_multiple_dirs.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}" +TRY="$TRY_TOP/try" + +cleanup() { + cd / + + if [ -d "$try_workspace" ] + then + rm -rf "$try_workspace" >/dev/null 2>&1 + fi + + if [ -f "$try_example_dir1" ] + then + rm "$try_example_dir1" + fi + + if [ -f "$try_example_dir2" ] + then + rm "$try_example_dir2" + fi + + if [ -f "$try_example_dir3" ] + then + rm "$try_example_dir3" + fi + + if [ -f "$expected1" ] + then + rm "$expected1" + fi + + if [ -f "$expected2" ] + then + rm "$expected2" + fi + if [ -f "$expected3" ] + then + rm "$expected3" + fi +} + +trap 'cleanup' EXIT + +try_workspace="$(mktemp -d -p .)" +cp "$TRY_TOP/test/resources/file.txt.gz" "$try_workspace/" +cd "$try_workspace" || return 1 + +try_example_dir1="$(mktemp -d)" +try_example_dir2="$(mktemp -d)" +try_example_dir3="$(mktemp -d)" + +expected1="$(mktemp)" +expected2="$(mktemp)" +expected3="$(mktemp)" + +touch "$expected1" +echo "test2" > "$expected2" +echo "test3" > "$expected3" + +"$TRY" -D "$try_example_dir1" "touch file_1.txt; echo test > file_2.txt; rm file.txt.gz" || return 2 +"$TRY" -D "$try_example_dir2" "echo test2 > file_2.txt" || return 3 +"$TRY" -D "$try_example_dir3" "echo test3 > file_3.txt" || return 4 +"$TRY" -L "$try_example_dir3:$try_example_dir2:$try_example_dir1" -y "cat file_1.txt > out1; cat file_2.txt > out2; cat file_3.txt > out3"|| return 5 + +diff -q "$expected1" out1 || return 6 +diff -q "$expected2" out2 || return 7 +diff -q "$expected3" out3 || return 8 + +! [ -f out4 ] || return 9 diff --git a/try b/try index 05768337..979118d0 100755 --- a/try +++ b/try @@ -45,7 +45,7 @@ try() { fi ## Make any directories that don't already exist, this is OK to do here - ## because we have already checked if it valid. + ## because we have already checked if it valid. export SANDBOX_DIR mkdir -p "$SANDBOX_DIR/upperdir" "$SANDBOX_DIR/workdir" "$SANDBOX_DIR/temproot" @@ -56,6 +56,34 @@ try() { findmnt --real -r -o target -n >>"$DIRS_AND_MOUNTS" sort -u -o "$DIRS_AND_MOUNTS" "$DIRS_AND_MOUNTS" + # Calculate UPDATED_DIRS_AND_MOUNTS that contains the merge arguments in LOWER_DIRS + UPDATED_DIRS_AND_MOUNTS="$(mktemp)" + export UPDATED_DIRS_AND_MOUNTS + while IFS="" read -r mountpoint + do + new_mountpoint="" + OLDIFS=$IFS + IFS=":" + + for lower_dir in $LOWER_DIRS + do + temp_mountpoint="$lower_dir/upperdir$mountpoint" + if [ -n "$new_mountpoint" ] + then + # If new_mountpoint is not empty, append : and the temp_mountpoint + new_mountpoint="$new_mountpoint:$temp_mountpoint" + else + # If new_mountpoint is empty, just set it to temp_mountpoint + new_mountpoint="$temp_mountpoint" + fi + done + IFS=$OLDIFS + # Add the original mountpoint at the end + new_mountpoint="${new_mountpoint:+$new_mountpoint:}$mountpoint" + echo "$new_mountpoint" >> "$UPDATED_DIRS_AND_MOUNTS" + done <"$DIRS_AND_MOUNTS" + + # we will overlay-mount each root directory separately (instead of all at once) because some directories cannot be overlayed # so we set up the mount points now # @@ -94,11 +122,12 @@ TRY_COMMAND="$TRY_COMMAND($0)" ## A wrapper of `mount -t overlay` to have cleaner looking code make_overlay() { sandbox_dir="$1" - lowerdir="$2" - mountpoint="$3" - mount -t overlay overlay -o userxattr -o "lowerdir=$lowerdir,upperdir=$sandbox_dir/upperdir/$mountpoint,workdir=$sandbox_dir/workdir/$mountpoint" "$sandbox_dir/temproot/$mountpoint" + lowerdirs="$2" + overlay_mountpoint="$3" + mount -t overlay overlay -o userxattr -o "lowerdir=$lowerdirs,upperdir=$sandbox_dir/upperdir/$overlay_mountpoint,workdir=$sandbox_dir/workdir/$overlay_mountpoint" "$sandbox_dir/temproot/$overlay_mountpoint" } + devices_to_mount="tty null zero full random urandom" ## Mounts and unmounts a few select devices instead of the whole `/dev` @@ -138,24 +167,23 @@ then fi # actually mount the overlays -for mountpoint in $(cat "$DIRS_AND_MOUNTS") +for mountpoint in $(cat "$UPDATED_DIRS_AND_MOUNTS") do + pure_mountpoint=${mountpoint##*:} + ## We are not interested in mounts that are not directories - if ! [ -d "$mountpoint" ] + if ! [ -d "$pure_mountpoint" ] then continue fi - ## Don't do anything for the root - ## and skip if it is /dev or /proc, we will mount it later - if [ "$mountpoint" = "/" ] || - [ "$mountpoint" = "/dev" ] || [ "$mountpoint" = "/proc" ] - then - continue - fi + ## Don't do anything for the root and skip if it is /dev or /proc, we will mount it later + case "$pure_mountpoint" in + (/|/dev|/proc) continue;; + esac # Try mounting everything normally - make_overlay "$SANDBOX_DIR" "/$mountpoint" "$mountpoint" 2>>"$try_mount_log" + make_overlay "$SANDBOX_DIR" "$mountpoint" "$pure_mountpoint" 2>>"$try_mount_log" # If mounting everything normally fails, we try using either using mergerfs or unionfs to mount them. if [ "$?" -ne 0 ] then @@ -180,9 +208,7 @@ do ## Create a union directory "$UNION_HELPER" $mountpoint $merger_dir 2>>"$try_mount_log" || printf "%s: Warning: Failed mounting $mountpoint via $UNION_HELPER, see \"$try_mount_log\"\n" "$TRY_COMMAND" >&2 - - ## Make the overlay on the union directory which works as a lowerdir for overlay - make_overlay "$SANDBOX_DIR" "$merger_dir" "$mountpoint" 2>>"$try_mount_log" || + make_overlay "$SANDBOX_DIR" "$merger_dir" "$pure_mountpoint" 2>>"$try_mount_log" || printf "%s: Warning: Failed mounting $mountpoint as an overlay via $UNION_HELPER, see \"$try_mount_log\"\n" "$TRY_COMMAND" >&2 fi fi @@ -477,7 +503,7 @@ error() { usage() { cat >&2 <&2 exit 0;; (U) if ! [ -x "$OPTARG" ]