fix: avoid redundant extraction / copies of tarballs / layouts

CHANGELOG.md

@@ -11,6 +11,12 @@
 - `singularity.conf` now accepts setting the `allow uts ns` option, and can
   invalidate the use of the `--uts` and `--hostname` flags.
 
+### Bug Fixes
+
+- Avoid unnecessary copying / extraction of OCI images and Docker tarballs into
+  a layout directory when they are directly accessible as a local file /
+  directory.
+
 ## 4.1.3 \[2024-05-08\]
 
 ### Requirements

internal/pkg/build/sources/conveyorPacker_oci.go

@@ -1,5 +1,5 @@
 // Copyright (c) 2020, Control Command Inc. All rights reserved.
-// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -159,8 +159,9 @@ func (cp *OCIConveyorPacker) Get(ctx context.Context, b *sytypes.Bundle) (err er
 		imgCache = cp.b.Opts.ImgCache
 	}
 
-	// Fetch the image into a temporary containers/image oci layout dir.
-	cp.srcImg, err = ociimage.FetchToLayout(ctx, cp.transportOptions, imgCache, ref, b.TmpDir)
+	// If the image is not a local file or OCI layout, fetch into cache or
+	// temporary layout.
+	cp.srcImg, err = ociimage.LocalImage(ctx, cp.transportOptions, imgCache, ref, b.TmpDir)
 	if err != nil {
 		return err
 	}

internal/pkg/client/ocisif/ocisif.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Sylabs Inc. All rights reserved.
+// Copyright (c) 2023-2024 Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -133,7 +133,7 @@ func createOciSif(ctx context.Context, tOpts *ociimage.TransportOptions, imgCach
 		return err
 	}
 
-	img, err := ociimage.FetchToLayout(ctx, tOpts, imgCache, imageSrc, tmpDir)
+	img, err := ociimage.LocalImage(ctx, tOpts, imgCache, imageSrc, tmpDir)
 	if err != nil {
 		return fmt.Errorf("while fetching OCI image: %w", err)
 	}

internal/pkg/ociimage/digest.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.

internal/pkg/ociimage/fetch.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved.
+// Copyright (c) 2019-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -49,20 +49,47 @@ func cachedImage(ctx context.Context, imgCache *cache.Handle, srcImg ggcrv1.Imag
 	return OCISourceSink.Image(ctx, cachedRef, nil, nil)
 }
 
-// FetchToLayout will fetch the OCI image specified by imageRef to an OCI layout
-// and return a v1.Image referencing it. If imgCache is non-nil, and enabled,
-// the image will be fetched into Singularity's cache - which is a multi-image
-// OCI layout. If the cache is disabled, the image will be fetched into a
-// subdirectory of the provided tmpDir. The caller is responsible for cleaning
-// up tmpDir.
-func FetchToLayout(ctx context.Context, tOpts *TransportOptions, imgCache *cache.Handle, imageURI, tmpDir string) (ggcrv1.Image, error) {
-	// oci-archive - Perform a tar extraction first, and handle as an oci layout.
+// LocalImage returns a ggrcv1.Image for imageURI that is guaranteed to be
+// backed by a local file or directory. If the image is an OCI layout or docker
+// tarball then it can be accessed directly. If the image is a tarball of an OCI
+// layout then it is extracted to tmpDir. If the image is remote, or in the
+// docker daemon, it will be pulled into the local cache - which is a
+// multi-image OCI layout. If the cache is disabled, the image will be fetched
+// into a subdirectory of the provided tmpDir. The caller is responsible for
+// cleaning up tmpDir.
+func LocalImage(ctx context.Context, tOpts *TransportOptions, imgCache *cache.Handle, imageURI, tmpDir string) (ggcrv1.Image, error) {
+	// oci-archive tarball is a local file, but current ggcr cannot read
+	// directly. Must always extract to a layoutdir .
 	if strings.HasPrefix(imageURI, "oci-archive:") {
-		layoutURI, cleanup, err := extractOCIArchive(imageURI, tmpDir)
+		return LocalImageLayout(ctx, tOpts, imgCache, imageURI, tmpDir)
+	}
+
+	srcType, srcRef, err := URItoSourceSinkRef(imageURI)
+	if err != nil {
+		return nil, err
+	}
+	// Docker tarballs and OCI layouts are already local
+	if srcType == TarballSourceSink || srcType == OCISourceSink {
+		return srcType.Image(ctx, srcRef, tOpts, nil)
+	}
+
+	return LocalImageLayout(ctx, tOpts, imgCache, imageURI, tmpDir)
+}
+
+// LocalImage returns a ggrcv1.Image for imageURI that is guaranteed to be
+// backed by a local OCI layout directory. If the image is a tarball of an OCI
+// layout then it is extracted to tmpDir. If the image is remote, or in the
+// docker daemon, it will be pulled into the local cache - which is a
+// multi-image OCI layout. If the cache is disabled, the image will be fetched
+// into a subdirectory of the provided tmpDir. The caller is responsible for
+// cleaning up tmpDir.
+func LocalImageLayout(ctx context.Context, tOpts *TransportOptions, imgCache *cache.Handle, imageURI, tmpDir string) (ggcrv1.Image, error) {
+	if strings.HasPrefix(imageURI, "oci-archive:") {
+		// oci-archive is a straight tar of an OCI layout, so extract to a tempDir
+		layoutURI, _, err := extractOCIArchive(imageURI, tmpDir)
 		if err != nil {
 			return nil, err
 		}
-		defer cleanup()
 		imageURI = layoutURI
 	}
 
@@ -71,8 +98,13 @@ func FetchToLayout(ctx context.Context, tOpts *TransportOptions, imgCache *cache
 		return nil, err
 	}
 
-	rt := progress.NewRoundTripper(ctx, nil)
+	// We might already have an OCI layout at this point
+	if srcType == OCISourceSink {
+		return srcType.Image(ctx, srcRef, tOpts, nil)
+	}
 
+	// Registry / Docker Daemon images need to be fetched
+	rt := progress.NewRoundTripper(ctx, nil)
 	srcImg, err := srcType.Image(ctx, srcRef, tOpts, rt)
 	if err != nil {
 		rt.ProgressShutdown()
@@ -108,8 +140,8 @@ func FetchToLayout(ctx context.Context, tOpts *TransportOptions, imgCache *cache
 }
 
 // extractOCIArchive will extract a tar `oci-archive:` image into a temporary
-// layout dir. The caller is responsible for calling cleanup() to remove the
-// temporary layout.
+// layout that will be a subdirectory of tmpDir. The caller is responsible for
+// calling cleanup to remove the layout, or otherwise cleaning up tmpDir.
 func extractOCIArchive(archiveURI, tmpDir string) (layoutURI string, cleanup func(), err error) {
 	layoutDir, err := os.MkdirTemp(tmpDir, "temp-oci-")
 	if err != nil {

pkg/ocibundle/native/bundle_linux.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved.
+// Copyright (c) 2022-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -162,20 +162,19 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error {
 		return fmt.Errorf("failed to generate OCI bundle/config: %s", err)
 	}
 
-	// Get a reference to an OCI layout source for the image, fetching the image
-	// through the cache if it is enabled.
+	// Ensure we have a reference to the image as a local file / layout,
+	// fetching the image through the cache / tmpdir if necessary.
 	tmpLayout, err := os.MkdirTemp(b.tmpDir, "oci-tmp-layout")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(tmpLayout)
-
-	layoutImg, err := ociimage.FetchToLayout(ctx, b.transportOptions, b.imgCache, b.imageRef, tmpLayout)
+	localImg, err := ociimage.LocalImage(ctx, b.transportOptions, b.imgCache, b.imageRef, tmpLayout)
 	if err != nil {
 		return err
 	}
 
-	manifestData, err := layoutImg.RawManifest()
+	manifestData, err := localImg.RawManifest()
 	if err != nil {
 		return fmt.Errorf("error obtaining manifest source: %s", err)
 	}
@@ -184,7 +183,7 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error {
 		return fmt.Errorf("error parsing manifest: %w", err)
 	}
 
-	configData, err := layoutImg.RawConfigFile()
+	configData, err := localImg.RawConfigFile()
 	if err != nil {
 		return fmt.Errorf("error obtaining image config source: %s", err)
 	}
@@ -202,7 +201,7 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error {
 	}
 	pristineRootfs := filepath.Join(b.rootfsParentDir, "rootfs")
 
-	if err := ociimage.UnpackRootfs(ctx, layoutImg, pristineRootfs); err != nil {
+	if err := ociimage.UnpackRootfs(ctx, localImg, pristineRootfs); err != nil {
 		return err
 	}