oci: buildkit TMPDIR fallback

Detect whether the `~/singularity.d/singularity-buildkit` location can
be used for buildkit. Full overlay support is required.

If not compatible, try falling back to a temporary directory with a
warning that there will be no persistent cache.

If TMPDIR is not compatible, give a sensible fatal error.

Fixes #3382

Author: dtrudg

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 - Fix regression from 4.1.5 that overwrites source image runscript, environment
   etc. in build from local image.
+- Fall back to `$TMPDIR` as singularity-buildkitd root directory if
+  `~/.singularity` is on a filesystem that does not fully support overlay.
 
 ### New Features & Functionality
 

cmd/singularity-buildkitd/main.go

@@ -9,21 +9,27 @@ import (
 	"context"
 	"os"
 
+	"github.com/spf13/pflag"
 	bkdaemon "github.com/sylabs/singularity/v4/internal/pkg/build/buildkit/daemon"
 	"github.com/sylabs/singularity/v4/internal/pkg/buildcfg"
 	"github.com/sylabs/singularity/v4/pkg/sylog"
 	"github.com/sylabs/singularity/v4/pkg/util/singularityconf"
 )
 
+var (
+	rootDir  string
+	arch     string
+	bkSocket string
+)
+
 func main() {
-	if len(os.Args) < 2 || len(os.Args) > 3 {
-		sylog.Fatalf("%s: usage: %s <socket-uri> [architecture]", bkdaemon.DaemonName, os.Args[0])
-	}
+	pflag.StringVar(&rootDir, "root", "", "buildkitd root directory")
+	pflag.StringVar(&arch, "arch", "", "build architecture")
+	pflag.StringVar(&bkSocket, "socket", "", "socket path")
+	pflag.Parse()
 
-	bkSocket := os.Args[1]
-	bkArch := ""
-	if len(os.Args) == 3 {
-		bkArch = os.Args[2]
+	if bkSocket == "" {
+		sylog.Fatalf("%s: usage: %s [--root <dir>] [--arch <arch>] --socket <socket-uri>", bkdaemon.DaemonName, os.Args[0])
 	}
 
 	sylog.Debugf("%s: parsing configuration file %s", bkdaemon.DaemonName, buildcfg.SINGULARITY_CONF_FILE)
@@ -34,8 +40,10 @@ func main() {
 	singularityconf.SetCurrentConfig(config)
 
 	daemonOpts := &bkdaemon.Opts{
-		ReqArch: bkArch,
+		ReqArch: arch,
+		RootDir: rootDir,
 	}
+
 	if err := bkdaemon.Run(context.Background(), daemonOpts, bkSocket); err != nil {
 		sylog.Fatalf("%s: %v", bkdaemon.DaemonName, err)
 	}

internal/pkg/build/buildkit/client/client.go

@@ -47,14 +47,16 @@ import (
 	"github.com/sylabs/singularity/v4/internal/pkg/ociplatform"
 	"github.com/sylabs/singularity/v4/internal/pkg/remote/credential/ociauth"
 	"github.com/sylabs/singularity/v4/internal/pkg/util/bin"
+	fsoverlay "github.com/sylabs/singularity/v4/internal/pkg/util/fs/overlay"
+	"github.com/sylabs/singularity/v4/pkg/syfs"
 	"github.com/sylabs/singularity/v4/pkg/sylog"
 	"golang.org/x/sync/errgroup"
 )
 
 const (
 	buildTag          = "tag"
 	bkDefaultSocket   = "unix:///run/buildkit/buildkitd.sock"
-	bkLaunchTimeout   = 120 * time.Second
+	bkLaunchTimeout   = 10 * time.Second
 	bkShutdownTimeout = 10 * time.Second
 	bkMinVersion      = "v0.12.3"
 )
@@ -163,11 +165,27 @@ func startBuildkitd(ctx context.Context, opts *Opts) (bkSocket string, cleanup f
 
 	bkSocket = generateSocketAddress()
 
-	// singularity-buildkitd <socket-uri> [architecture]
-	args := []string{bkSocket}
+	args := []string{}
+	tmpRoot := ""
+	// Check the user .singularity dir is in a location supporting overlayfs etc. If not, use a tmpdir.
+	if err := fsoverlay.CheckUpper(syfs.ConfigDir()); err != nil {
+		tmpRoot, err = os.MkdirTemp("", "singularity-buildkitd-")
+		if err != nil {
+			sylog.Fatalf("while creating singularity-buildkitd temporary root dir: %v", err)
+		}
+		if err := fsoverlay.CheckUpper(tmpRoot); err != nil {
+			sylog.Fatalf("Temporary directory does not support buildkit. Please set $TMPDIR to a local filesystem.")
+		}
+
+		sylog.Warningf("~/.singularity filesystem does not support buildkit. Using temporary directory %s. Layers will not be cached for future builds.", tmpRoot)
+		args = append(args, "--root="+tmpRoot)
+	}
+
 	if opts.ReqArch != "" {
-		args = append(args, opts.ReqArch)
+		args = append(args, "--arch="+opts.ReqArch)
 	}
+	args = append(args, "--socket="+bkSocket)
+
 	cmd := exec.CommandContext(ctx, bkCmd, args...)
 	cmd.WaitDelay = bkShutdownTimeout
 	cmd.Cancel = func() error {
@@ -182,8 +200,15 @@ func startBuildkitd(ctx context.Context, opts *Opts) (bkSocket string, cleanup f
 			sylog.Errorf("while canceling buildkit daemon process: %v", err)
 		}
 		cmd.Wait()
+		if tmpRoot != "" {
+			sylog.Warningf("removing singularity-buildkitd temporary directory %s", tmpRoot)
+			if err := os.RemoveAll(tmpRoot); err != nil {
+				sylog.Errorf("while removing singularity-buildkitd temp dir: %v", err)
+			}
+		}
 	}
 
+	sylog.Debugf("starting %s %v", bkCmd, args)
 	if err := cmd.Start(); err != nil {
 		return "", nil, err
 	}

internal/pkg/build/buildkit/daemon/daemon.go

@@ -80,6 +80,8 @@ const DaemonName = "singularity-buildkitd"
 type Opts struct {
 	// Requested build architecture
 	ReqArch string
+	// Override the location of the singularity-buildkitd root with specified directory
+	RootDir string
 }
 
 type workerInitializerOpt struct {
@@ -149,7 +151,7 @@ func waitLock(ctx context.Context, lockPath string) (*flock.Flock, error) {
 func Run(ctx context.Context, opts *Opts, socketPath string) error {
 	// If we need to, enter a new cgroup now, to workaround an issue with crun container cgroup creation (#1538).
 	if err := oci.CrunNestCgroup(); err != nil {
-		sylog.Fatalf("%s: while applying crun cgroup workaround: %v", DaemonName, err)
+		return fmt.Errorf("%s: while applying crun cgroup workaround: %v", DaemonName, err)
 	}
 
 	cfg, err := config.LoadFile(defaultConfigPath())
@@ -166,6 +168,14 @@ func Run(ctx context.Context, opts *Opts, socketPath string) error {
 
 	server := grpc.NewServer()
 
+	if opts.RootDir != "" {
+		ptr := func(v bool) *bool {
+			return &v
+		}
+		cfg.Root = opts.RootDir
+		cfg.Workers.OCI.GC = ptr(false)
+	}
+
 	// relative path does not work with nightlyone/lockfile
 	root, err := filepath.Abs(cfg.Root)
 	if err != nil {
@@ -181,7 +191,7 @@ func Run(ctx context.Context, opts *Opts, socketPath string) error {
 	sylog.Debugf("%s: path for buildkitd lock file: %s", DaemonName, lockPath)
 	lock, err := waitLock(ctx, lockPath)
 	if err != nil {
-		sylog.Fatalf("%s: while creating lock file: %v", DaemonName, err)
+		return fmt.Errorf("%s: while creating lock file: %v", DaemonName, err)
 	}
 	defer func() {
 		lock.Unlock()