fuse: automatically use squashfuse for images, deprecate --sif-fuse

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Changes Since Last Release
 
+### Changed defaults / behaviours
+
+- In native mode, SIF/SquashFS container images will now be mounted with
+  squashfuse when kernel mounts are disabled in `singularity.conf`, or cannot be
+  used (non-setuid / user namespace workflow). If the FUSE mount fails,
+  Singularity will fall back to extracting the container to a temporary sandbox
+  in order to run it.
+
 ### New Features & Functionality
 
 - The `registry login` and `registry logout` commands now support a `--authfile
@@ -27,10 +35,18 @@
   executable, then the `run` / `exec` / `shell` commands in `--oci` mode can be
   given the `--app <appname>` flag, and will automatically invoke the relevant
   SCIF command.
-- SIF/SquashFS container images can now be mounted using FUSE in all native mode
-  flows, including setuid mode. To enable, use the `--sif-fuse` flag, or set
-  `sif fuse = yes` in `singularity.conf`. Overlay partitions and extfs images
-  are not yet supported.
+- A new `--tmp-sandbox` flag has been added to the `run / shell / exec /
+  instance start` commands. This will force Singularity to extract a container
+  to a temporary sandbox before running it, when it would otherwise perform a
+  kernel or FUSE mount.
+
+### Deprecated Functionality
+
+- The experimental `--sif-fuse` flag, and `sif fuse` directive in
+  `singularity.conf` are deprecated. The flag and directive were used to enable
+  experimental mounting of SIF/SquashFS container images with FUSE in prior
+  versions of Singularity. From 4.1, FUSE mounts are used automatically when
+  kernel mounts are disabled / not available.
 
 ### Bug Fixes
 

cmd/internal/cli/action_flags.go

@@ -766,6 +766,7 @@ var actionSIFFUSEFlag = cmdline.Flag{
 	Name:         "sif-fuse",
 	Usage:        "attempt FUSE mount of SIF",
 	EnvKeys:      []string{"SIF_FUSE"},
+	Deprecated:   "FUSE mounts are now used automatically when kernel mounts are disabled / unavailable.",
 }
 
 // --proot (hidden)
@@ -891,6 +892,7 @@ func init() {
 		cmdManager.RegisterFlagForCmd(&commonOCIFlag, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&commonNoOCIFlag, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&commonKeepLayersFlag, actionsInstanceCmd...)
+		cmdManager.RegisterFlagForCmd(&actionTmpSandbox, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&actionNoTmpSandbox, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&actionDevice, actionsCmd...)

cmd/internal/cli/actions.go

@@ -387,6 +387,7 @@ func launchContainer(cmd *cobra.Command, ep launcher.ExecParams) error {
 		launcher.OptDevice(device),
 		launcher.OptCdiDirs(cdiDirs),
 		launcher.OptNoCompat(noCompat),
+		launcher.OptTmpSandbox(tmpSandbox),
 		launcher.OptNoTmpSandbox(noTmpSandbox),
 	}
 

cmd/internal/cli/singularity.go

@@ -84,6 +84,7 @@ var (
 
 	// Options controlling the unpacking of images to temporary sandboxes
 	canUseTmpSandbox bool
+	tmpSandbox       bool
 	noTmpSandbox     bool
 
 	// Use OCI runtime and OCI SIF?
@@ -305,6 +306,16 @@ var commonKeepLayersFlag = cmdline.Flag{
 	EnvKeys:      []string{"KEEP_LAYERS"},
 }
 
+// --tmp-sandbox
+var actionTmpSandbox = cmdline.Flag{
+	ID:           "actionTmpSandbox",
+	Value:        &tmpSandbox,
+	DefaultValue: false,
+	Name:         "tmp-sandbox",
+	Usage:        "Forces unpacking of images into temporary sandbox dirs when a kernel or FUSE mount would otherwise be used.",
+	EnvKeys:      []string{"TMP_SANDBOX"},
+}
+
 // --no-tmp-sandbox
 var actionNoTmpSandbox = cmdline.Flag{
 	ID:           "actionNoTmpSandbox",

e2e/actions/actions.go

@@ -2507,8 +2507,10 @@ func (c actionTests) actionCompat(t *testing.T) {
 	}
 }
 
-// actionSquashfuse tests that squashfuse SIF mount works.
-func (c actionTests) actionSIFFUSE(t *testing.T) {
+// actionFUSEImage tests that squashfuse SIF mount works. Currently forced here
+// via deprecated `--sif-fuse` flag as this is convenient to include non-userns
+// profiles without changing global config.
+func (c actionTests) actionFUSEImage(t *testing.T) {
 	require.Command(t, "squashfuse")
 	require.Command(t, "fusermount")
 	e2e.EnsureImage(t, c.env)
@@ -2521,11 +2523,14 @@ func (c actionTests) actionSIFFUSE(t *testing.T) {
 			e2e.AsSubtest(p.String()),
 			e2e.WithProfile(e2e.UserNamespaceProfile),
 			e2e.WithCommand("exec"),
+			e2e.WithGlobalOptions("-d"),
 			e2e.WithArgs("--sif-fuse", c.env.ImagePath, "ps"),
 			e2e.ExpectExit(
 				0,
 				e2e.ExpectOutput(e2e.ContainMatch, "squashfuse"),
 				e2e.ExpectError(e2e.ContainMatch, "Mounting image with FUSE"),
+				e2e.ExpectError(e2e.ContainMatch, "PostStartHost()"),
+				e2e.ExpectError(e2e.ContainMatch, "CleanupHost()"),
 			),
 		)
 
@@ -2536,45 +2541,59 @@ func (c actionTests) actionSIFFUSE(t *testing.T) {
 	}
 }
 
-// Verify that the FUSE mounts, and the CleanupHost() process are not seen when
-// --sif-fuse should not be in effect.
-func (c actionTests) actionNoSIFFUSE(t *testing.T) {
+// Verify that the FUSE mounts, and the PostStartHost/CleanupHost() processes are not seen when
+// FUSE mounts of a SIF image should not be in effect.
+func (c actionTests) actionNoFUSEImage(t *testing.T) {
 	e2e.EnsureImage(t, c.env)
 
-	for _, p := range e2e.NativeProfiles {
+	for _, p := range []e2e.Profile{e2e.RootProfile, e2e.UserProfile} {
 		c.env.RunSingularity(
 			t,
 			e2e.AsSubtest(p.String()),
 			e2e.WithProfile(p),
 			e2e.WithCommand("exec"),
 			e2e.WithGlobalOptions("-d"),
-			e2e.WithArgs(c.env.ImagePath, "mount"),
+			e2e.WithArgs(c.env.ImagePath, "ps"),
 			e2e.ExpectExit(
 				0,
 				e2e.ExpectError(e2e.UnwantedContainMatch, "squashfuse"),
+				e2e.ExpectError(e2e.UnwantedContainMatch, "PostStartHost()"),
 				e2e.ExpectError(e2e.UnwantedContainMatch, "CleanupHost()"),
 			),
 		)
 	}
 }
 
-// actionTmpSandboxFlag tests the command-line option prohibiting unpacking of image
+// actionTmpSandboxFlag tests the command-line options forcing / prohibiting unpacking of image
 // files into temporary sandbox dirs.
 func (c actionTests) actionTmpSandboxFlag(t *testing.T) {
 	e2e.EnsureImage(t, c.env)
 
-	profiles := []e2e.Profile{e2e.UserProfile, e2e.RootProfile, e2e.FakerootProfile, e2e.UserNamespaceProfile}
-
-	for _, p := range profiles {
-		c.env.RunSingularity(
-			t,
-			e2e.AsSubtest(p.String()),
-			e2e.WithProfile(p),
-			e2e.WithCommand("exec"),
-			e2e.WithArgs("--sif-fuse=false", "--no-tmp-sandbox", "-u", c.env.ImagePath, "/bin/true"),
-			e2e.ExpectExit(255),
-		)
+	// --tmp-sandbox should override kernel mount (setuid profiles) and squashfuse mount (userns profiles).
+	for _, p := range e2e.NativeProfiles {
+		t.Run(p.String(), func(t *testing.T) {
+			c.env.RunSingularity(
+				t,
+				e2e.AsSubtest("tmp-sandbox"),
+				e2e.WithProfile(p),
+				e2e.WithCommand("exec"),
+				e2e.WithArgs("--tmp-sandbox", c.env.ImagePath, "/bin/sh", "-c", "echo $SINGULARITY_CONTAINER"),
+				e2e.ExpectExit(0,
+					e2e.ExpectOutput(e2e.RegexMatch, `/rootfs-(\d+)/root`), // <tmpdir>/rootfs-xxxxxxxxx/root
+					e2e.ExpectError(e2e.ContainMatch, "Converting SIF file to temporary sandbox"),
+				),
+			)
+		})
 	}
+
+	c.env.RunSingularity(
+		t,
+		e2e.AsSubtest("no-tmp-sandbox"),
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("exec"),
+		e2e.WithArgs("--tmp-sandbox", "--no-tmp-sandbox", c.env.ImagePath, "/bin/sh", "-c", "echo $SINGULARITY_CONTAINER"),
+		e2e.ExpectExit(255),
+	)
 }
 
 // Make sure --workdir and --scratch work together nicely even when workdir is a
@@ -2831,9 +2850,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 		"compat":                       np(c.actionCompat),               // test --compat
 		"umask":                        np(c.actionUmask),                // test umask propagation
 		"invalidRemote":                np(c.invalidRemote),              // GHSA-5mv9-q7fq-9394
-		"SIFFUSE":                      np(c.actionSIFFUSE),              // test --sif-fuse
-		"NoSIFFUSE":                    np(c.actionNoSIFFUSE),            // test absence of squashfs and CleanupHost()
-		"TmpSandboxFlag":               c.actionTmpSandboxFlag,           // test --no-tmp-sandbox flag
+		"FUSEImage":                    np(c.actionFUSEImage),            // test explicit FUSE image mount
+		"NoFUSEImage":                  np(c.actionNoFUSEImage),          // test absence of squashfuse and CleanupHost()
+		"TmpSandboxFlag":               c.actionTmpSandboxFlag,           // test --tmp-sandbox / --no-tmp-sandbox flag
 		"relWorkdirScratch":            np(c.relWorkdirScratch),          // test relative --workdir with --scratch
 		"ociRelWorkdirScratch":         np(c.actionOciRelWorkdirScratch), // test relative --workdir with --scratch in OCI mode
 		"auth":                         np(c.actionAuth),                 // tests action cmds w/authenticated pulls from OCI registries

e2e/config/config.go

@@ -533,7 +533,8 @@ func (c configTests) configGlobal(t *testing.T) {
 			profile:        e2e.UserProfile,
 			directive:      "allow kernel squashfs",
 			directiveValue: "no",
-			exit:           255,
+			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.ContainMatch, "Mounting image with FUSE"),
 		},
 		{
 			name:           "AllowKernelSquashfsYes_Container",
@@ -542,6 +543,7 @@ func (c configTests) configGlobal(t *testing.T) {
 			directive:      "allow kernel squashfs",
 			directiveValue: "yes",
 			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.UnwantedContainMatch, "Mounting image with FUSE"),
 		},
 		// Standalone ext3 rootfs
 		{
@@ -635,7 +637,8 @@ func (c configTests) configGlobal(t *testing.T) {
 			profile:        e2e.UserProfile,
 			directive:      "allow kernel squashfs",
 			directiveValue: "no",
-			exit:           255,
+			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.ContainMatch, "Mounting image with FUSE"),
 		},
 		{
 			name:           "AllowKernelSquashfsYes_SIF",
@@ -644,6 +647,7 @@ func (c configTests) configGlobal(t *testing.T) {
 			directive:      "allow kernel squashfs",
 			directiveValue: "yes",
 			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.UnwantedContainMatch, "Mounting image with FUSE"),
 		},
 		// Encrypted squashFS rootfs in SIF
 		{

internal/app/starter/host.go

@@ -9,12 +9,13 @@ import (
 	"context"
 	"net"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/sylabs/singularity/v4/internal/pkg/runtime/engine"
 	"github.com/sylabs/singularity/v4/pkg/sylog"
 )
 
-//nolint:dupl
 func PostStartHost(postStartSocket int, e *engine.Engine) {
 	sylog.Debugf("Entering PostStartHost")
 	comm := os.NewFile(uintptr(postStartSocket), "unix")
@@ -30,7 +31,7 @@ func PostStartHost(postStartSocket int, e *engine.Engine) {
 	// Wait for a write into the socket from master to trigger after container process started.
 	data := make([]byte, 1)
 	if _, err := conn.Read(data); err != nil {
-		sylog.Fatalf("While reading from cleanup socket: %s", err)
+		sylog.Fatalf("While reading from post start socket: %s", err)
 	}
 
 	if err := e.PostStartHost(ctx); err != nil {
@@ -47,9 +48,17 @@ func PostStartHost(postStartSocket int, e *engine.Engine) {
 	os.Exit(0)
 }
 
-//nolint:dupl
 func CleanupHost(cleanupSocket int, e *engine.Engine) {
 	sylog.Debugf("Entering CleanupHost")
+
+	// An unclean shutdown, in which the parent of the cleanup process exits,
+	// will result in SIGTERM (as this was set as the parent death signal in the
+	// C starter code).
+	tc := make(chan os.Signal, 1)
+	signal.Notify(tc, syscall.SIGTERM)
+
+	// A clean container shutdown results in the master process sending a
+	// message on the cleanup socket to trigger cleanup. We will use SIGHUP to indicate this.
 	comm := os.NewFile(uintptr(cleanupSocket), "unix")
 	conn, err := net.FileConn(comm)
 	if err != nil {
@@ -58,24 +67,48 @@ func CleanupHost(cleanupSocket int, e *engine.Engine) {
 	comm.Close()
 	defer conn.Close()
 
-	ctx := context.TODO()
+	go func() {
+		data := make([]byte, 1)
+		_, err := conn.Read(data)
+		// Clean shutdown - master should be notified after cleanup.
+		if err == nil {
+			tc <- syscall.SIGHUP
+		}
+		// Unclean shutdown - master not available to notify after cleanup.
+		tc <- syscall.SIGTERM
+		sylog.Debugf("While reading from cleanup socket: %s", err)
+	}()
 
-	// Wait for a write into the socket from master to trigger cleanup after container exited
-	data := make([]byte, 1)
-	if _, err := conn.Read(data); err != nil {
-		sylog.Fatalf("While reading from cleanup socket: %s", err)
-	}
+	// Block here until direct SIGTERM, or generated SIGHUP from reading master
+	// socket, is received.
+	sig := <-tc
+	sylog.Debugf("CleanupHost Signaled: %v", sig)
+
+	// Run engine specific cleanup tasks
+	err = e.CleanupHost(context.TODO())
 
-	if err := e.CleanupHost(ctx); err != nil {
+	// If we are in a clean (master initiated) shutdown, notify master of any cleanup failure.
+	if err != nil && sig == syscall.SIGHUP {
 		if _, err := conn.Write([]byte{'f'}); err != nil {
-			sylog.Fatalf("Could not write to master: %s", err)
+			sylog.Debugf("Could not write to master: %s", err)
 		}
-		sylog.Fatalf("While running host cleanup tasks: %s", err)
+	}
+	// Exit on cleanup failure.
+	if err != nil {
+		sylog.Errorf("While running host cleanup tasks: %s", err)
+		sylog.Debugf("Exiting CleanupHost - Cleanup failure")
+		os.Exit(1)
 	}
 
-	if _, err := conn.Write([]byte{'c'}); err != nil {
-		sylog.Fatalf("Could not write to master: %s", err)
+	// If we are in a clean (master initiated) shutdown, notify master of the cleanup success.
+	if sig == syscall.SIGHUP {
+		if _, err := conn.Write([]byte{'c'}); err != nil {
+			sylog.Debugf("Could not write to master: %s", err)
+			sylog.Debugf("Exiting CleanupHost - Master socket write failure")
+			os.Exit(1)
+		}
 	}
-	sylog.Debugf("Exiting CleanupHost")
+
+	sylog.Debugf("Exiting CleanupHost - Success")
 	os.Exit(0)
 }

internal/pkg/runtime/launcher/native/launcher_linux.go

@@ -977,16 +977,17 @@ func (l *Launcher) setCgroups(instanceName string) error {
 	return nil
 }
 
-// PrepareImage perfoms any image preparation required before execution.
-// This is currently limited to extraction or FUSE mount when using the user namespace,
-// and activating any image driver plugins that might handle the image mount.
+// PrepareImage performs any image preparation required before execution.
+// This is currently limited to extraction or FUSE mount.
 func (l *Launcher) prepareImage(c context.Context, image string) error {
 	if strings.HasPrefix(image, "instance://") {
 		return nil
 	}
 
 	insideUserNs, _ := namespaces.IsInsideUserNamespace(os.Getpid())
 	isUserNs := insideUserNs || l.cfg.Namespaces.User
+	noKernelMount := l.cfg.TmpSandbox || isUserNs || l.cfg.SIFFUSE
+	tryFuse := !l.cfg.TmpSandbox
 
 	img, err := imgutil.Init(image, false)
 	if err != nil {
@@ -1002,20 +1003,20 @@ func (l *Launcher) prepareImage(c context.Context, image string) error {
 	case imgutil.SANDBOX:
 		return nil
 	case imgutil.SQUASHFS:
-		if isUserNs || l.cfg.SIFFUSE {
-			return l.prepareSquashfs(c, img, l.cfg.SIFFUSE)
+		if !l.engineConfig.File.AllowKernelSquashfs || noKernelMount {
+			return l.prepareSquashfs(c, img, tryFuse)
 		}
 		// setuid, kernel squashfs permitted, fuse not requested - no action needed
 		return nil
 	case imgutil.ENCRYPTSQUASHFS:
-		if isUserNs || l.cfg.SIFFUSE {
+		if !l.engineConfig.File.AllowKernelSquashfs || noKernelMount {
 			return fmt.Errorf("encrypted SIF files are only supported in setuid mode, with kernel mounts")
 		}
 		// setuid, kernel squashfs permitted, fuse not requested - no action needed
 		return nil
 	case imgutil.EXT3:
 		if isUserNs || l.cfg.SIFFUSE {
-			return l.prepareExtfs(c, img, l.cfg.SIFFUSE)
+			return l.prepareExtfs(c, img)
 		}
 		// setuid, kernel extfs permitted, fuse not requested - no action needed
 		return nil
@@ -1069,7 +1070,7 @@ func (l *Launcher) prepareSquashfs(ctx context.Context, img *imgutil.Image, tryF
 	return fmt.Errorf("extraction failed: %v", err)
 }
 
-func (l *Launcher) prepareExtfs(_ context.Context, _ *imgutil.Image, _ bool) error {
+func (l *Launcher) prepareExtfs(_ context.Context, _ *imgutil.Image) error {
 	// TODO - Enable fuse2fs handling
 	return fmt.Errorf("extfs images can only be run in setuid mode with kernel extfs mounts enabled")
 }