feat: oci-sif overlay digest sync

CHANGELOG.md

@@ -24,6 +24,10 @@
   overlay create` command. The overlay will be applied read-only, by default,
   when executing the OCI-SIF. To write changes to the container into the overlay,
   use the `--writable` flag.
+- A writable overlay is added to an OCI-SIF file as an ext3 format layer,
+  appended to the encapsulated OCI image. After the overlay has been modified,
+  use the `singularity overlay sync` command to synchronize the OCI digests with
+  the overlay content.
 - Added a new `instance run` command that will execute the runscript when an
   instance is initiated instead of executing the startscript.
 

cmd/internal/cli/overlay.go

@@ -11,11 +11,13 @@ import (
 func init() {
 	addCmdInit(func(cmdManager *cmdline.CommandManager) {
 		cmdManager.RegisterCmd(OverlayCmd)
-		cmdManager.RegisterSubCmd(OverlayCmd, OverlayCreateCmd)
 
+		cmdManager.RegisterSubCmd(OverlayCmd, OverlayCreateCmd)
 		cmdManager.RegisterFlagForCmd(&overlaySizeFlag, OverlayCreateCmd)
 		cmdManager.RegisterFlagForCmd(&overlayCreateDirFlag, OverlayCreateCmd)
 		cmdManager.RegisterFlagForCmd(&overlaySparseFlag, OverlayCreateCmd)
+
+		cmdManager.RegisterSubCmd(OverlayCmd, OverlaySyncCmd)
 	})
 }
 

cmd/internal/cli/overlay_sync.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2024, Sylabs Inc. All rights reserved.
+// This software is licensed under a 3-clause BSD license. Please consult the
+// LICENSE.md file distributed with the sources of this project regarding your
+// rights to use or distribute this software.
+package cli
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/sylabs/singularity/v4/docs"
+	"github.com/sylabs/singularity/v4/internal/pkg/ocisif"
+	"github.com/sylabs/singularity/v4/pkg/sylog"
+)
+
+// OverlaySyncCmd is the 'overlay sync' command that updates the digest of
+// an overlay in an OCI-SIF image, as recorded in the OCI manifest / config.
+var OverlaySyncCmd = &cobra.Command{
+	Args: cobra.ExactArgs(1),
+	RunE: func(_ *cobra.Command, args []string) error {
+		if err := ocisif.SyncOverlay(args[0]); err != nil {
+			sylog.Fatalf(err.Error())
+		}
+		return nil
+	},
+	DisableFlagsInUseLine: true,
+
+	Use:     docs.OverlaySyncUse,
+	Short:   docs.OverlaySyncShort,
+	Long:    docs.OverlaySyncLong,
+	Example: docs.OverlaySyncExample,
+}

docs/content.go

@@ -1143,6 +1143,15 @@ Enterprise Performance Computing (EPC)`
   To create a sparse overlay when creating a new ext3 file system image:
   $ singularity overlay create --size 1024 --sparse /tmp/ext3_overlay.img`
 
+	OverlaySyncUse   string = `sync oci-sif`
+	OverlaySyncShort string = `Sync OCI-SIF manifest & config with overlay content`
+	OverlaySyncLong  string = `
+  The overlay sync command updates the OCI manifest and config in an OCI-SIF image
+  to reflect any changes that have been made to the content of a writable overlay.`
+	OverlaySyncExample string = `
+  To synchronize an OCI-SIF image containing an overlay:
+  $ singularity overlay sync /tmp/overlay.oci.sif`
+
 	DataUse   string = `data`
 	DataShort string = `Manage an OCI-SIF data container`
 	DataLong  string = `

e2e/overlay/overlay.go

@@ -294,6 +294,14 @@ func (c ctx) testOverlayOCI(t *testing.T) {
 			args:    []string{ocisif, "ls", "/in-overlay"},
 			exit:    0,
 		},
+		// Synchronize the overlay digest.
+		{
+			name:    "sync",
+			profile: e2e.UserProfile,
+			command: "overlay",
+			args:    []string{"sync", ocisif},
+			exit:    0,
+		},
 		// Remove file without `--writable` - should be an ephemeral change, file still in overlay.
 		{
 			name:    "tmpfs rm",

internal/pkg/ocisif/overlay.go

@@ -14,44 +14,49 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/types"
+	ocitmutate "github.com/sylabs/oci-tools/pkg/mutate"
 	ocitsif "github.com/sylabs/oci-tools/pkg/sif"
 	"github.com/sylabs/sif/v2/pkg/sif"
 	"github.com/sylabs/singularity/v4/pkg/image"
+	"github.com/sylabs/singularity/v4/pkg/sylog"
 )
 
 var Ext3LayerMediaType types.MediaType = "application/vnd.sylabs.image.layer.v1.ext3"
 
-// HasOverlay returns whether the OCI-SIF at imgPath has an ext3 writable final
-// layer - an 'overlay'. If present, the offset of the overlay data in the
-// OCI-SIF file is also returned.
-func HasOverlay(imagePath string) (bool, int64, error) {
-	fi, err := sif.LoadContainerFromPath(imagePath,
-		sif.OptLoadWithFlag(os.O_RDONLY),
-	)
-	if err != nil {
-		return false, 0, err
-	}
-	defer fi.UnloadContainer()
-
+func getSingleImage(fi *sif.FileImage) (v1.Image, error) {
 	ii, err := ocitsif.ImageIndexFromFileImage(fi)
 	if err != nil {
-		return false, 0, fmt.Errorf("while obtaining image index: %w", err)
+		return nil, fmt.Errorf("while obtaining image index: %w", err)
 	}
 	ix, err := ii.IndexManifest()
 	if err != nil {
-		return false, 0, fmt.Errorf("while obtaining index manifest: %w", err)
+		return nil, fmt.Errorf("while obtaining index manifest: %w", err)
 	}
 
 	// One image only.
 	if len(ix.Manifests) != 1 {
-		return false, 0, fmt.Errorf("only single image data containers are supported, found %d images", len(ix.Manifests))
+		return nil, fmt.Errorf("only single image data containers are supported, found %d images", len(ix.Manifests))
 	}
 	imageDigest := ix.Manifests[0].Digest
-	img, err := ii.Image(imageDigest)
+	return ii.Image(imageDigest)
+}
+
+// HasOverlay returns whether the OCI-SIF at imgPath has an ext3 writable final
+// layer - an 'overlay'. If present, the offset of the overlay data in the
+// OCI-SIF file is also returned.
+func HasOverlay(imagePath string) (bool, int64, error) {
+	fi, err := sif.LoadContainerFromPath(imagePath,
+		sif.OptLoadWithFlag(os.O_RDONLY),
+	)
 	if err != nil {
-		return false, 0, fmt.Errorf("while initializing image: %w", err)
+		return false, 0, err
 	}
+	defer fi.UnloadContainer()
 
+	img, err := getSingleImage(fi)
+	if err != nil {
+		return false, 0, fmt.Errorf("while getting image: %w", err)
+	}
 	layers, err := img.Layers()
 	if err != nil {
 		return false, 0, fmt.Errorf("while getting image layers: %w", err)
@@ -89,40 +94,99 @@ func AddOverlay(imagePath string, overlayPath string) error {
 	}
 	defer fi.UnloadContainer()
 
-	ii, err := ocitsif.ImageIndexFromFileImage(fi)
+	img, err := getSingleImage(fi)
 	if err != nil {
-		return fmt.Errorf("while obtaining image index: %w", err)
+		return fmt.Errorf("while getting image: %w", err)
 	}
-	ix, err := ii.IndexManifest()
+
+	ol, err := ext3LayerFromFile(overlayPath)
 	if err != nil {
-		return fmt.Errorf("while obtaining index manifest: %w", err)
-	}
-	if len(ix.Manifests) != 1 {
-		return fmt.Errorf("only single image OCI-SIF files are supported - found %d images", len(ix.Manifests))
+		return err
 	}
-	imageDigest := ix.Manifests[0].Digest
-	img, err := ii.Image(imageDigest)
+
+	img, err = mutate.AppendLayers(img, ol)
 	if err != nil {
-		return fmt.Errorf("while initializing image: %w", err)
+		return err
 	}
 
-	ol, err := ext3LayerFromFile(overlayPath)
+	ii := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: img})
+
+	return ocitsif.Update(fi, ii)
+}
+
+// SyncOverlay synchronizes the digests of the overlay, stored in the OCI
+// structures, with its true content.
+func SyncOverlay(imagePath string) error {
+	fi, err := sif.LoadContainerFromPath(imagePath)
 	if err != nil {
 		return err
 	}
+	defer fi.UnloadContainer()
 
-	img, err = mutate.AppendLayers(img, ol)
+	img, err := getSingleImage(fi)
+	if err != nil {
+		return fmt.Errorf("while getting image: %w", err)
+	}
+	layers, err := img.Layers()
+	if err != nil {
+		return fmt.Errorf("while getting image layers: %w", err)
+	}
+	if len(layers) < 1 {
+		return fmt.Errorf("image has no layers")
+	}
+	oldLayer := layers[len(layers)-1]
+	mt, err := oldLayer.MediaType()
+	if err != nil {
+		return fmt.Errorf("while getting layer mediatype: %w", err)
+	}
+	if mt != Ext3LayerMediaType {
+		return fmt.Errorf("image does not contain a writable overlay")
+	}
+
+	// Existing descriptor and digest
+	oldDigest, err := oldLayer.Digest()
+	if err != nil {
+		return err
+	}
+	desc, err := fi.GetDescriptor(sif.WithOCIBlobDigest(oldDigest))
+	if err != nil {
+		return err
+	}
+	// Updated layer and digest
+	o := func() (io.ReadCloser, error) {
+		return io.NopCloser(desc.GetReader()), nil
+	}
+	newLayer, err := ext3LayerFromOpener(o)
+	if err != nil {
+		return err
+	}
+	newDigest, err := newLayer.Digest()
 	if err != nil {
 		return err
 	}
 
-	ii = mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: img})
+	if newDigest == oldDigest {
+		sylog.Infof("OCI digest matches overlay, no update required.")
+		return nil
+	}
 
+	// Update overlay OCI.Blob digest in SIF. This must be done before the
+	// oci-tools Update is called, so that it re-uses the existing overlay
+	// descriptor, with the updated digest.
+	if err := fi.SetOCIBlobDigest(desc.ID(), newDigest); err != nil {
+		return fmt.Errorf("while updating descriptor digest: %v", err)
+	}
+
+	img, err = ocitmutate.Apply(img, ocitmutate.SetLayer(len(layers)-1, newLayer))
+	if err != nil {
+		return err
+	}
+	ii := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: img})
 	return ocitsif.Update(fi, ii)
 }
 
 // ext3ImageOpener opens a source ext3 overlay image file to be added as a layer.
-type ext3ImageOpener func() (io.ReadSeekCloser, error)
+type ext3ImageOpener func() (io.ReadCloser, error)
 
 type ext3ImageLayer struct {
 	opener ext3ImageOpener
@@ -173,7 +237,7 @@ func (l *ext3ImageLayer) MediaType() (types.MediaType, error) {
 }
 
 func ext3LayerFromFile(path string) (v1.Layer, error) {
-	opener := func() (io.ReadSeekCloser, error) {
+	opener := func() (io.ReadCloser, error) {
 		return os.Open(path)
 	}
 	return ext3LayerFromOpener(opener)
@@ -198,10 +262,17 @@ func ext3LayerFromOpener(opener ext3ImageOpener) (v1.Layer, error) {
 		return nil, fmt.Errorf("while checking overlay file header: %w", err)
 	}
 
-	_, err = rc.Seek(0, io.SeekStart)
+	// Re-open rather than seek, so we can use the SIF GetReader API which
+	// returns an io.Reader only.
+	if err := rc.Close(); err != nil {
+		return nil, err
+	}
+	rc, err = opener()
 	if err != nil {
 		return nil, err
 	}
+	defer rc.Close()
+
 	digest, size, err := v1.SHA256(rc)
 	if err != nil {
 		return nil, err