From 71cb5699aae8b27b0347de5e5ebf234e402e90a3 Mon Sep 17 00:00:00 2001 From: ezri Date: Tue, 17 Sep 2024 23:17:58 -0400 Subject: [PATCH 1/5] stdstream support (#168) * stdstream support * stdstream test (using /dev/std{in,out,err}), and fds directly --- test/stdstream.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++++ try | 7 +++++++ 2 files changed, 58 insertions(+) create mode 100755 test/stdstream.sh diff --git a/test/stdstream.sh b/test/stdstream.sh new file mode 100755 index 00000000..e7a3e22c --- /dev/null +++ b/test/stdstream.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree 2>/dev/null || echo "${0%/*}")}" +TRY="$TRY_TOP/try" + +cmdfile="$(mktemp)" + +cat > "$cmdfile" <<'EOF' +read x < /dev/stdin +echo $((x * 2)) > /dev/stdout +echo $((x * 3)) > /dev/stderr + +EOF + +chmod +x "$cmdfile" + +try_stdout=$(mktemp) +try_stderr=$(mktemp) +sh_stdout=$(mktemp) +sh_stderr=$(mktemp) + +# test stdout +echo 5 | "$TRY" "$cmdfile" >"$try_stdout" 2>"$try_stderr" +echo 5 | sh "$cmdfile" >"$sh_stdout" 2>"$sh_stderr" + +diff "$try_stdout" "$sh_stdout" || exit 1 + +# using grep because there's try errors printed +grep -q 15 "$try_stderr" +grep -q 15 "$sh_stderr" + +rm "$try_stdout" "$try_stderr" "$sh_stdout" "$sh_stderr" + +cat > "$cmdfile" <<'EOF' +read x <&0 +echo $((x * 2)) >&1 +echo $((x * 3)) >&2 + +EOF + +# test stdout +echo 5 | "$TRY" "$cmdfile" >"$try_stdout" 2>"$try_stderr" +echo 5 | sh "$cmdfile" >"$sh_stdout" 2>"$sh_stderr" + +diff "$try_stdout" "$sh_stdout" || exit 1 + +# using grep because there's try errors printed +grep -q 15 "$try_stderr" +grep -q 15 "$sh_stderr" + +rm "$try_stdout" "$try_stderr" "$sh_stdout" "$sh_stderr" diff --git a/try b/try index 09e8c08b..64495aa2 100755 --- a/try +++ b/try @@ -249,6 +249,10 @@ unshare --root="$SANDBOX_DIR/temproot" /bin/sh "$chroot_executable" exitcode="$?" # unmount the devices +rm "$sandbox_dir/temproot/dev/stdin" +rm "$sandbox_dir/temproot/dev/stdout" +rm "$sandbox_dir/temproot/dev/stderr" + unmount_devices "$SANDBOX_DIR" exit $exitcode @@ -262,6 +266,9 @@ unset START_DIR SANDBOX_DIR UNION_HELPER DIRS_AND_MOUNTS TRY_EXIT_STATUS unset script_to_execute chroot_executable try_mount_log mount -t proc proc /proc && +ln -s /proc/self/fd/0 /dev/stdin && +ln -s /proc/self/fd/1 /dev/stdout && +ln -s /proc/self/fd/2 /dev/stderr && cd "$START_DIR" && . "$script_to_execute" EOF From 939f3ee80fada8199a1bc4d91acbaf9010a2c3a0 Mon Sep 17 00:00:00 2001 From: Michael Greenberg Date: Thu, 10 Oct 2024 09:43:34 -0400 Subject: [PATCH 2/5] try to get unshare from util-linux (#177) merging because CI is broken and GH is not debugging properly --- .github/workflows/test.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fd4ae70e..22122b3a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,7 +17,12 @@ jobs: steps: - name: Install dependencies run: | - sudo apt-get install expect mergerfs attr pandoc + sudo apt-get install util-linux expect mergerfs attr pandoc + + - name: Debug unshare install + run: | + echo unshare is at $(which unshare) + unshare --mount --map-root-user --user --pid --fork -- ls - name: Checkout uses: actions/checkout@v4 From 18a365c397e21cead99f1816d1a20139ecbc211f Mon Sep 17 00:00:00 2001 From: Michael Greenberg Date: Thu, 10 Oct 2024 13:20:05 -0400 Subject: [PATCH 3/5] turn on unprivileged user namespaces in CI (#178) Ubuntu 24.04 LTS has this feature turned off, but we need it to configure the build or run tests --- .github/workflows/test.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 22122b3a..b74cf0b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,14 +15,13 @@ jobs: if: github.event.pull_request.draft == false steps: - - name: Install dependencies + - name: Allow unprivileged user namespaces (for Ubuntu 24.04) run: | - sudo apt-get install util-linux expect mergerfs attr pandoc + sudo sysctl kernel.apparmor_restrict_unprivileged_userns=0 - - name: Debug unshare install + - name: Install dependencies run: | - echo unshare is at $(which unshare) - unshare --mount --map-root-user --user --pid --fork -- ls + sudo apt-get install util-linux expect mergerfs attr pandoc - name: Checkout uses: actions/checkout@v4 @@ -55,6 +54,10 @@ jobs: if: github.event.pull_request.draft == false steps: + - name: Allow unprivileged user namespaces (for Ubuntu 24.04) + run: | + sudo sysctl kernel.apparmor_restrict_unprivileged_userns=0 + - name: Install dependencies run: | sudo apt-get install expect mergerfs attr pandoc @@ -97,6 +100,10 @@ jobs: if: github.event.pull_request.draft == false steps: + - name: Allow unprivileged user namespaces (for Ubuntu 24.04) + run: | + sudo sysctl kernel.apparmor_restrict_unprivileged_userns=0 + - name: Install dependencies run: | sudo apt-get install expect mergerfs attr pandoc From 872845f0d439b57649400e3939682f54da8f61cc Mon Sep 17 00:00:00 2001 From: ezri Date: Tue, 29 Oct 2024 14:16:18 -0400 Subject: [PATCH 4/5] Named tmpfiles (#180) * name try's tmpfiles with current time as ID for uniqueness --- try | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/try b/try index 64495aa2..5774ae34 100755 --- a/try +++ b/try @@ -9,6 +9,8 @@ TRY_VERSION="0.2.0" TRY_COMMAND="${0##*/}" +EXECID="$(date +%s%3N)" +export EXECID export TRY_COMMAND # exit status invariants @@ -35,7 +37,7 @@ try() { ! [ -d "$SANDBOX_DIR" ] && { error "could not find sandbox directory $SANDBOX_DIR" 2; } else ## Create a new sandbox if one was not given - SANDBOX_DIR=$(mktemp -d) + SANDBOX_DIR=$(mktemp -d --suffix ".try-$EXECID") fi ## If the sandbox is not valid we exit early @@ -48,7 +50,7 @@ try() { ## because we have already checked if it valid. export SANDBOX_DIR - try_mount_log="$(mktemp)" + try_mount_log="$(mktemp --suffix ".try-$EXECID")" export try_mount_log # If we're in a docker container, we want to mount tmpfs on sandbox_dir, #136 @@ -64,14 +66,14 @@ try() { mkdir -p "$SANDBOX_DIR/upperdir" "$SANDBOX_DIR/workdir" "$SANDBOX_DIR/temproot" ## Find all the directories and mounts that need to be mounted - DIRS_AND_MOUNTS="$(mktemp)" + DIRS_AND_MOUNTS="$(mktemp --suffix ".try-$EXECID")" export DIRS_AND_MOUNTS find / -maxdepth 1 >"$DIRS_AND_MOUNTS" findmnt --real -r -o target -n >>"$DIRS_AND_MOUNTS" sort -u -o "$DIRS_AND_MOUNTS" "$DIRS_AND_MOUNTS" # Calculate UPDATED_DIRS_AND_MOUNTS that contains the merge arguments in LOWER_DIRS - UPDATED_DIRS_AND_MOUNTS="$(mktemp)" + UPDATED_DIRS_AND_MOUNTS="$(mktemp --suffix ".try-$EXECID")" export UPDATED_DIRS_AND_MOUNTS while IFS="" read -r mountpoint do @@ -122,9 +124,9 @@ try() { chmod "$(stat -c %a /)" "$SANDBOX_DIR/temproot" - mount_and_execute="$(mktemp)" - chroot_executable="$(mktemp)" - script_to_execute="$(mktemp)" + mount_and_execute="$(mktemp --suffix ".try-$EXECID")" + chroot_executable="$(mktemp --suffix ".try-$EXECID")" + script_to_execute="$(mktemp --suffix ".try-$EXECID")" export chroot_executable export script_to_execute @@ -225,7 +227,7 @@ do ## We can ignore this mountpoint, if the user program tries to use it, it will crash, but if not we can run normally printf "%s: Warning: Failed mounting $mountpoint as an overlay and mergerfs or unionfs not set and could not be found, see \"$try_mount_log\"\n" "$TRY_COMMAND" >&2 else - merger_dir=$(mktemp -d) + merger_dir=$(mktemp -d --suffix ".try-$EXECID") ## Create a union directory "$UNION_HELPER" $mountpoint $merger_dir 2>>"$try_mount_log" || @@ -610,7 +612,7 @@ EOF NO_COMMIT="interactive" # Includes all patterns given using the `-i` flag; will be used with `grep -f` -IGNORE_FILE="$(mktemp)" +IGNORE_FILE="$(mktemp --suffix ".try-$EXECID")" while getopts ":yvnhxi:D:U:L:" opt do From b63d1e7d3aafa7cd334a68d7077ce13ffa530d27 Mon Sep 17 00:00:00 2001 From: Michael Greenberg Date: Wed, 20 Nov 2024 07:06:05 -0500 Subject: [PATCH 5/5] Minimize tempfile churn (#186) Closes #185. Try was making way too many tempfiles. Now we store everything in the sandbox, with a tiny bit of nuance: $IGNORE_FILE needs to be created in advance to handle the args properly. So we create that temporary unconditionally. When running a command (the try() function), we'll move $IGNORE_FILE into the sandbox. When running try commit or try summary, we just delete $IGNORE_FILE at the end. --- test/tempfiles.sh | 30 ++++++++++++++++++++++++++++++ try | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 11 deletions(-) create mode 100755 test/tempfiles.sh diff --git a/test/tempfiles.sh b/test/tempfiles.sh new file mode 100755 index 00000000..25e6adb0 --- /dev/null +++ b/test/tempfiles.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# shellcheck disable=SC2010,SC2126,SC2181 + +TRY_TOP="${TRY_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree 2>/dev/null || echo "${0%/*}")}" +TRY="$TRY_TOP/try" + +workdir="$(mktemp -d)" +cd "$workdir" || exit 1 + +initial_count="$(ls "${TMPDIR-/tmp}" | grep -e "^.*\.try-[0-9]*$" | wc -l)" + +sandbox=$($TRY -n "touch $HOME/foo") +[ $? -eq 0 ] || exit 2 + +post_count="$(ls "${TMPDIR-/tmp}" | grep -e "^.*\.try-[0-9]*$" | wc -l)" + +# just one new tempfile +[ "$((initial_count + 1))" -eq "$post_count" ] || exit 3 +[ -f "$sandbox/upperdir$HOME/foo" ] || exit 4 + +# deliberately not the pattern of try sandboxes +sandbox=local +mkdir "$sandbox" || exit 5 +$TRY -D "$sandbox" "touch $HOME/bar" || exit 6 + +final_count="$(ls "${TMPDIR-/tmp}" | grep -e "^.*\.try-[0-9]*$" | wc -l)" + +# no new tempfiles! +[ "$post_count" -eq "$final_count" ] || exit 7 +[ -f "$sandbox/upperdir$HOME/bar" ] || exit 8 diff --git a/try b/try index 5774ae34..3f8e059d 100755 --- a/try +++ b/try @@ -34,10 +34,15 @@ try() { if [ "$SANDBOX_DIR" ] then ## If the name of a sandbox is given then we need to exit prematurely if its directory doesn't exist - ! [ -d "$SANDBOX_DIR" ] && { error "could not find sandbox directory $SANDBOX_DIR" 2; } + [ -d "$SANDBOX_DIR" ] || error "could not find sandbox directory $SANDBOX_DIR" 2 + # Force absolute path + SANDBOX_DIR="$(cd "$SANDBOX_DIR" && pwd)" + + # shellcheck disable=SC2181 + [ "$?" -eq 0 ] || error "could not find sandbox directory $SANDBOX_DIR (could not cd in)" 2 else ## Create a new sandbox if one was not given - SANDBOX_DIR=$(mktemp -d --suffix ".try-$EXECID") + SANDBOX_DIR="$(mktemp -d --suffix ".try-$EXECID")" fi ## If the sandbox is not valid we exit early @@ -50,7 +55,11 @@ try() { ## because we have already checked if it valid. export SANDBOX_DIR - try_mount_log="$(mktemp --suffix ".try-$EXECID")" + # We created "$IGNORE_FILE" up front, but now we can stash it in the sandbox. + mv "$IGNORE_FILE" "$SANDBOX_DIR"/ignore + IGNORE_FILE="$SANDBOX_DIR"/ignore + + try_mount_log="$SANDBOX_DIR"/mount.log export try_mount_log # If we're in a docker container, we want to mount tmpfs on sandbox_dir, #136 @@ -66,14 +75,14 @@ try() { mkdir -p "$SANDBOX_DIR/upperdir" "$SANDBOX_DIR/workdir" "$SANDBOX_DIR/temproot" ## Find all the directories and mounts that need to be mounted - DIRS_AND_MOUNTS="$(mktemp --suffix ".try-$EXECID")" + DIRS_AND_MOUNTS="$SANDBOX_DIR"/mounts export DIRS_AND_MOUNTS find / -maxdepth 1 >"$DIRS_AND_MOUNTS" findmnt --real -r -o target -n >>"$DIRS_AND_MOUNTS" sort -u -o "$DIRS_AND_MOUNTS" "$DIRS_AND_MOUNTS" # Calculate UPDATED_DIRS_AND_MOUNTS that contains the merge arguments in LOWER_DIRS - UPDATED_DIRS_AND_MOUNTS="$(mktemp --suffix ".try-$EXECID")" + UPDATED_DIRS_AND_MOUNTS="$SANDBOX_DIR"/mounts.updated export UPDATED_DIRS_AND_MOUNTS while IFS="" read -r mountpoint do @@ -124,9 +133,9 @@ try() { chmod "$(stat -c %a /)" "$SANDBOX_DIR/temproot" - mount_and_execute="$(mktemp --suffix ".try-$EXECID")" - chroot_executable="$(mktemp --suffix ".try-$EXECID")" - script_to_execute="$(mktemp --suffix ".try-$EXECID")" + mount_and_execute="$SANDBOX_DIR"/mount_and_execute.sh + chroot_executable="$SANDBOX_DIR"/chroot_executable.sh + script_to_execute="$SANDBOX_DIR"/script_to_execute.sh export chroot_executable export script_to_execute @@ -227,7 +236,8 @@ do ## We can ignore this mountpoint, if the user program tries to use it, it will crash, but if not we can run normally printf "%s: Warning: Failed mounting $mountpoint as an overlay and mergerfs or unionfs not set and could not be found, see \"$try_mount_log\"\n" "$TRY_COMMAND" >&2 else - merger_dir=$(mktemp -d --suffix ".try-$EXECID") + merger_dir="$SANDBOX_DIR"/mergerdir."$(echo "$pure_mountpoint" | tr '/' '.')" + mkdir "$merger_dir" ## Create a union directory "$UNION_HELPER" $mountpoint $merger_dir 2>>"$try_mount_log" || @@ -612,6 +622,9 @@ EOF NO_COMMIT="interactive" # Includes all patterns given using the `-i` flag; will be used with `grep -f` +# +# We have to create this temporary up front. +# We move it to $SANDBOX_DIR/ignore in `try()`, but delete it when we don't move it. IGNORE_FILE="$(mktemp --suffix ".try-$EXECID")" while getopts ":yvnhxi:D:U:L:" opt @@ -657,9 +670,13 @@ fi TRY_EXIT_STATUS=1 case "$1" in (summary) : "${SANDBOX_DIR=$2}" - summary;; + summary + rm "$IGNORE_FILE" # we didn't move it to the sandbox, so clean up + ;; (commit) : "${SANDBOX_DIR=$2}" - commit;; + commit + rm "$IGNORE_FILE" # we didn't move it to the sandbox, so clean up + ;; (explore) : "${SANDBOX_DIR=$2}" try "$SHELL";; (--) shift