Augment k0s reset integration tests

Add a script to the reset integration test that tries to clutter the
data directory by placing files and directories with odd permissions,
adding symlinks to stuff outside the data directory, and adding bind
mounts in various ways.

This is to prove that k0's reset will never delete anything that is not
beneath the data directory.

Signed-off-by: Tom Wieczorek <twieczorek@mirantis.com>

inttest/reset/clutter-data-dir.sh

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2024 k0s authors
+#shellcheck shell=ash
+
+set -eu
+
+make_dir() { mkdir -- "$1" && echo "$1"; }
+make_file() { echo "$1" >"$1" && echo "$1"; }
+
+make_bind_mounts() {
+  local real="$1"
+  local target="$2"
+
+  # Directory bind mount
+  make_dir "$real/real_dir"
+  make_file "$real/real_dir/real_dir_info.txt"
+  make_dir "$target/bind_dir"
+  mount --bind -- "$real/real_dir" "$target/bind_dir"
+
+  # File bind mount
+  make_file "$real/real_file.txt"
+  make_file "$target/bind_file.txt"
+  mount --bind -- "$real/real_file.txt" "$target/bind_file.txt"
+
+  # Recursive directory bind mount
+  make_dir "$real/real_recursive_dir"
+  make_file "$real/real_recursive_dir/real_recursive_dir.txt"
+  make_dir "$real/real_recursive_dir/bind_dir"
+  mount --bind -- "$real/real_dir" "$real/real_recursive_dir/bind_dir"
+  make_file "$real/real_recursive_dir/bind_file.txt"
+  mount --bind -- "$real/real_file.txt" "$real/real_recursive_dir/bind_file.txt"
+  make_dir "$target/rbind_dir"
+  mount --rbind -- "$real/real_recursive_dir" "$target/rbind_dir"
+
+  # Directory overmounts
+  make_dir "$real/overmount_dir"
+  make_file "$real/overmount_dir/in_overmount_dir.txt"
+  mount --bind -- "$real/overmount_dir" "$target/bind_dir"
+
+  # File overmounts
+  make_file "$real/overmount_file.txt"
+  mount --bind -- "$real/overmount_file.txt" "$target/bind_file.txt"
+}
+
+clutter() {
+  local dataDir="$1"
+  local realDir
+
+  realDir="$(mktemp -t -d k0s_reset_inttest.XXXXXX)"
+
+  local dir="$dataDir"/cluttered
+  make_dir "$dir"
+
+  # Directories and files with restricted permissions
+  make_dir "$dir/restricted_dir"
+  make_file "$dir/restricted_dir/no_read_file.txt"
+  chmod 000 -- "$dir/restricted_dir/no_read_file.txt" # No permissions on the file
+  make_dir "$dir/restricted_dir/no_exec_dir"
+  chmod 000 -- "$dir/restricted_dir/no_exec_dir" # No permissions on the directory
+  make_dir "$dir/restricted_dir/no_exec_nonempty_dir"
+  make_file "$dir/restricted_dir/no_exec_nonempty_dir/.hidden_file"
+  chmod 000 -- "$dir/restricted_dir/no_exec_nonempty_dir" # No permissions on the directory
+
+  # Symlinks pointing outside the directory tree
+  make_dir "$realDir/some_dir"
+  make_file "$realDir/some_dir/real_file.txt"
+  ln -s -- "$realDir/some_dir/real_file.txt" "$dir/symlink_to_file" # Symlink to a file
+  ln -s -- "$realDir/some_dir" "$dir/symlink_to_dir"                # Symlink to a directory
+
+  # Bind mounts pointing outside the directory tree
+  make_bind_mounts "$realDir" "$dir"
+
+  # Bind mounts outside the directory tree pointing into it
+  # make_bind_mounts "$dir" "$realDir"
+}
+
+clutter "$@"

inttest/reset/reset_test.go

@@ -17,6 +17,11 @@ limitations under the License.
 package reset
 
 import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+	"strings"
 	"testing"
 
 	testifysuite "github.com/stretchr/testify/suite"
@@ -28,11 +33,14 @@ type suite struct {
 	common.BootlooseSuite
 }
 
+//go:embed clutter-data-dir.sh
+var clutterScript []byte
+
 func (s *suite) TestReset() {
 	ctx := s.Context()
 	workerNode := s.WorkerNode(0)
 
-	if ok := s.Run("k0s gets up", func() {
+	if !s.Run("k0s gets up", func() {
 		s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server"))
 		s.Require().NoError(s.RunWorkers())
 
@@ -44,11 +52,7 @@ func (s *suite) TestReset() {
 
 		s.T().Log("waiting to see CNI pods ready")
 		s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start")
-	}); !ok {
-		return
-	}
 
-	s.Run("k0s reset", func() {
 		ssh, err := s.SSH(ctx, workerNode)
 		s.Require().NoError(err)
 		defer ssh.Disconnect()
@@ -57,14 +61,47 @@ func (s *suite) TestReset() {
 		s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory")
 
 		s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims")
+	}) {
+		return
+	}
 
+	var clutteringPaths bytes.Buffer
+
+	if !s.Run("prepare k0s reset", func() {
 		s.NoError(s.StopWorker(workerNode), "Failed to stop k0s")
 
+		ssh, err := s.SSH(ctx, workerNode)
+		s.Require().NoError(err)
+		defer ssh.Disconnect()
+
+		streams, flushStreams := common.TestLogStreams(s.T(), "clutter data dir")
+		streams.In = bytes.NewReader(clutterScript)
+		streams.Out = io.MultiWriter(&clutteringPaths, streams.Out)
+		err = ssh.Exec(ctx, "sh -s -- /var/lib/k0s", streams)
+		flushStreams()
+		s.Require().NoError(err)
+	}) {
+		return
+	}
+
+	s.Run("k0s reset", func() {
+		ssh, err := s.SSH(ctx, workerNode)
+		s.Require().NoError(err)
+		defer ssh.Disconnect()
+
 		streams, flushStreams := common.TestLogStreams(s.T(), "reset")
 		err = ssh.Exec(ctx, "k0s reset --debug", streams)
 		flushStreams()
 		s.NoError(err, "k0s reset didn't exit cleanly")
 
+		for _, path := range strings.Split(string(bytes.TrimSpace(clutteringPaths.Bytes())), "\n") {
+			if strings.HasPrefix(path, "/var/lib/k0s") {
+				s.NoError(ssh.Exec(ctx, fmt.Sprintf("! test -e %q", path), common.SSHStreams{}), "Failed to verify non-existence of %s", path)
+			} else {
+				s.NoError(ssh.Exec(ctx, fmt.Sprintf("test -e %q", path), common.SSHStreams{}), "Failed to verify existence of %s", path)
+			}
+		}
+
 		// /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty
 		s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty")
 		s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists")