From 7a4fc928e8e9785ff31c94433eacc66b6105d047 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Thu, 7 Nov 2024 17:52:48 +0100 Subject: [PATCH] Augment k0s reset integration tests Add a script to the reset integration test that tries to clutter the data directory by placing files and directories with odd permissions, adding symlinks to stuff outside the data directory, and adding bind mounts in various ways. This is to prove that k0's reset will never delete anything that is not beneath the data directory. Signed-off-by: Tom Wieczorek --- inttest/reset/clutter-data-dir.sh | 77 +++++++++++++++++++++++++++++++ inttest/reset/reset_test.go | 47 +++++++++++++++++-- 2 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 inttest/reset/clutter-data-dir.sh diff --git a/inttest/reset/clutter-data-dir.sh b/inttest/reset/clutter-data-dir.sh new file mode 100644 index 000000000000..c7c8f1e14d4a --- /dev/null +++ b/inttest/reset/clutter-data-dir.sh @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2024 k0s authors +#shellcheck shell=ash + +set -eu + +make_dir() { mkdir -- "$1" && echo "$1"; } +make_file() { echo "$1" >"$1" && echo "$1"; } + +make_bind_mounts() { + local real="$1" + local target="$2" + + # Directory bind mount + make_dir "$real/real_dir" + make_file "$real/real_dir/real_dir_info.txt" + make_dir "$target/bind_dir" + mount --bind -- "$real/real_dir" "$target/bind_dir" + + # File bind mount + make_file "$real/real_file.txt" + make_file "$target/bind_file.txt" + mount --bind -- "$real/real_file.txt" "$target/bind_file.txt" + + # Recursive directory bind mount + make_dir "$real/real_recursive_dir" + make_file "$real/real_recursive_dir/real_recursive_dir.txt" + make_dir "$real/real_recursive_dir/bind_dir" + mount --bind -- "$real/real_dir" "$real/real_recursive_dir/bind_dir" + make_file "$real/real_recursive_dir/bind_file.txt" + mount --bind -- "$real/real_file.txt" "$real/real_recursive_dir/bind_file.txt" + make_dir "$target/rbind_dir" + mount --rbind -- "$real/real_recursive_dir" "$target/rbind_dir" + + # Directory overmounts + make_dir "$real/overmount_dir" + make_file "$real/overmount_dir/in_overmount_dir.txt" + mount --bind -- "$real/overmount_dir" "$target/bind_dir" + + # File overmounts + make_file "$real/overmount_file.txt" + mount --bind -- "$real/overmount_file.txt" "$target/bind_file.txt" +} + +clutter() { + local dataDir="$1" + local realDir + + realDir="$(mktemp -t -d k0s_reset_inttest.XXXXXX)" + + local dir="$dataDir"/cluttered + make_dir "$dir" + + # Directories and files with restricted permissions + make_dir "$dir/restricted_dir" + make_file "$dir/restricted_dir/no_read_file.txt" + chmod 000 -- "$dir/restricted_dir/no_read_file.txt" # No permissions on the file + make_dir "$dir/restricted_dir/no_exec_dir" + chmod 000 -- "$dir/restricted_dir/no_exec_dir" # No permissions on the directory + make_dir "$dir/restricted_dir/no_exec_nonempty_dir" + make_file "$dir/restricted_dir/no_exec_nonempty_dir/.hidden_file" + chmod 000 -- "$dir/restricted_dir/no_exec_nonempty_dir" # No permissions on the directory + + # Symlinks pointing outside the directory tree + make_dir "$realDir/some_dir" + make_file "$realDir/some_dir/real_file.txt" + ln -s -- "$realDir/some_dir/real_file.txt" "$dir/symlink_to_file" # Symlink to a file + ln -s -- "$realDir/some_dir" "$dir/symlink_to_dir" # Symlink to a directory + + # Bind mounts pointing outside the directory tree + make_bind_mounts "$realDir" "$dir" + + # Bind mounts outside the directory tree pointing into it + # make_bind_mounts "$dir" "$realDir" +} + +clutter "$@" diff --git a/inttest/reset/reset_test.go b/inttest/reset/reset_test.go index e34fcef8d989..d6e5d2a6712d 100644 --- a/inttest/reset/reset_test.go +++ b/inttest/reset/reset_test.go @@ -17,6 +17,11 @@ limitations under the License. package reset import ( + "bytes" + _ "embed" + "fmt" + "io" + "strings" "testing" testifysuite "github.com/stretchr/testify/suite" @@ -28,11 +33,14 @@ type suite struct { common.BootlooseSuite } +//go:embed clutter-data-dir.sh +var clutterScript []byte + func (s *suite) TestReset() { ctx := s.Context() workerNode := s.WorkerNode(0) - if ok := s.Run("k0s gets up", func() { + if !s.Run("k0s gets up", func() { s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server")) s.Require().NoError(s.RunWorkers()) @@ -44,11 +52,7 @@ func (s *suite) TestReset() { s.T().Log("waiting to see CNI pods ready") s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start") - }); !ok { - return - } - s.Run("k0s reset", func() { ssh, err := s.SSH(ctx, workerNode) s.Require().NoError(err) defer ssh.Disconnect() @@ -57,14 +61,47 @@ func (s *suite) TestReset() { s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory") s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims") + }) { + return + } + var clutteringPaths bytes.Buffer + + if !s.Run("prepare k0s reset", func() { s.NoError(s.StopWorker(workerNode), "Failed to stop k0s") + ssh, err := s.SSH(ctx, workerNode) + s.Require().NoError(err) + defer ssh.Disconnect() + + streams, flushStreams := common.TestLogStreams(s.T(), "clutter data dir") + streams.In = bytes.NewReader(clutterScript) + streams.Out = io.MultiWriter(&clutteringPaths, streams.Out) + err = ssh.Exec(ctx, "sh -s -- /var/lib/k0s", streams) + flushStreams() + s.Require().NoError(err) + }) { + return + } + + s.Run("k0s reset", func() { + ssh, err := s.SSH(ctx, workerNode) + s.Require().NoError(err) + defer ssh.Disconnect() + streams, flushStreams := common.TestLogStreams(s.T(), "reset") err = ssh.Exec(ctx, "k0s reset --debug", streams) flushStreams() s.NoError(err, "k0s reset didn't exit cleanly") + for _, path := range strings.Split(string(bytes.TrimSpace(clutteringPaths.Bytes())), "\n") { + if strings.HasPrefix(path, "/var/lib/k0s") { + s.NoError(ssh.Exec(ctx, fmt.Sprintf("! test -e %q", path), common.SSHStreams{}), "Failed to verify non-existence of %s", path) + } else { + s.NoError(ssh.Exec(ctx, fmt.Sprintf("test -e %q", path), common.SSHStreams{}), "Failed to verify existence of %s", path) + } + } + // /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty") s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists")