fix: gracefully handle manifests missing from storage (prepare for sparse indexes)

GC and scrub should not stop if a manifest or index is missing from storage.
Other similar changes are also included.

WRT metadb, the missing manifests cannot be added, and the results returned from metadb
do not include the descriptors for these manifests.

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

Author: andaaron

pkg/extensions/search/cve/scan_test.go

@@ -384,6 +384,11 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo
 					return false, err
 				}
 
+				// If all manifests are missing (e.g., from an index), Manifests will be empty
+				if len(manifestData.Manifests) == 0 {
+					return false, nil
+				}
+
 				for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers {
 					switch imageLayer.MediaType {
 					case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer):

pkg/extensions/search/search_test.go

@@ -7406,6 +7406,98 @@ type repoRef struct {
 	Tag  string
 }
 
+func TestSearchWithMissingManifest(t *testing.T) {
+	Convey("Search with missing manifest", t, func() {
+		dir := t.TempDir()
+
+		// 1. Write the image to the disk
+		log := log.NewTestLogger()
+		storeCtlr := ociutils.GetDefaultStoreController(dir, log)
+
+		// Create a multiarch image with exactly 2 manifests
+		multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+
+		// Write the multiarch image to filesystem
+		err := WriteMultiArchImageToFileSystem(multiarchImage, "testrepo", "latest", storeCtlr)
+		So(err, ShouldBeNil)
+
+		// Get the image store to access index and manifests
+		imageStore := storeCtlr.GetDefaultImageStore()
+
+		// Get the index content to find all manifest digests
+		indexBlob, err := imageStore.GetIndexContent("testrepo")
+		So(err, ShouldBeNil)
+
+		var indexContent ispec.Index
+		err = json.Unmarshal(indexBlob, &indexContent)
+		So(err, ShouldBeNil)
+
+		So(len(indexContent.Manifests), ShouldBeGreaterThanOrEqualTo, 2)
+
+		// Get the first manifest digest to delete
+		firstManifestDigest := indexContent.Manifests[0].Digest
+
+		// Get the second manifest digest (should remain valid)
+		secondManifestDigest := indexContent.Manifests[1].Digest
+
+		// 2. Delete the manifest from the disk
+		manifestBlobPath := path.Join(dir, "testrepo", "blobs", "sha256", firstManifestDigest.Encoded())
+		err = os.Remove(manifestBlobPath)
+		So(err, ShouldBeNil)
+
+		// 3. Start the controller (MetaDB parsing would be done in the background)
+		port := GetFreePort()
+		baseURL := GetBaseURL(port)
+		conf := config.New()
+		conf.HTTP.Port = port
+		conf.Storage.RootDirectory = dir
+		defaultVal := true
+		conf.Extensions = &extconf.ExtensionConfig{
+			Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
+		}
+
+		conf.Extensions.Search.CVE = nil
+
+		ctlr := api.NewController(conf)
+
+		ctlrManager := NewControllerManager(ctlr)
+		ctlrManager.StartAndWait(port)
+		defer ctlrManager.StopServer()
+
+		// Search for the repository
+		query := `
+		{
+			GlobalSearch(query:"testrepo:latest"){
+				Images {
+					RepoName Tag
+					Manifests {
+						Digest
+					}
+				}
+			}
+		}`
+
+		resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
+		So(resp, ShouldNotBeNil)
+		So(err, ShouldBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+		responseStruct := &zcommon.GlobalSearchResultResp{}
+		err = json.Unmarshal(resp.Body(), responseStruct)
+		So(err, ShouldBeNil)
+
+		// Verify we found the image
+		So(responseStruct.GlobalSearchResult.GlobalSearch.Images, ShouldNotBeEmpty)
+		foundImage := responseStruct.GlobalSearchResult.GlobalSearch.Images[0]
+		So(foundImage.RepoName, ShouldEqual, "testrepo")
+		So(foundImage.Tag, ShouldEqual, "latest")
+
+		// Verify only the valid manifest is found in search results (missing one was skipped by ParseStorage)
+		So(len(foundImage.Manifests), ShouldEqual, 1)
+		So(foundImage.Manifests[0].Digest, ShouldEqual, secondManifestDigest.String())
+	})
+}
+
 func deleteUsedImages(repoTags []repoRef, baseURL string) {
 	for _, image := range repoTags {
 		status, err := DeleteImage(image.Repo, image.Tag, baseURL)

pkg/extensions/sync/destination.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/distribution/distribution/v3/registry/storage/driver"
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/regclient/regclient/types/mediatype"
@@ -227,11 +228,27 @@ func (registry *DestinationRegistry) copyManifest(repo string, desc ispec.Descri
 			return err
 		}
 
+		var firstMissingErr error
+
 		for _, manifest := range indexManifest.Manifests {
 			reference := GetDescriptorReference(manifest)
 
 			manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest)
 			if err != nil {
+				// Handle missing manifest blobs gracefully - log warning and continue with other manifests
+				var pathNotFoundErr driver.PathNotFoundError
+				if errors.Is(err, zerr.ErrBlobNotFound) || errors.As(err, &pathNotFoundErr) {
+					if firstMissingErr == nil {
+						firstMissingErr = err
+					}
+
+					registry.log.Warn().Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).
+						Str("digest", manifest.Digest.String()).
+						Msg("skipping missing manifest blob in image index, continuing sync with other manifests")
+
+					continue
+				}
+
 				registry.log.Error().Str("errorType", common.TypeOf(err)).
 					Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()).
 					Msg("failed find manifest which is part of an image index")
@@ -254,6 +271,11 @@ func (registry *DestinationRegistry) copyManifest(repo string, desc ispec.Descri
 			}
 		}
 
+		// Return error if we encountered any missing manifests
+		if firstMissingErr != nil {
+			return firstMissingErr
+		}
+
 		_, _, err := imageStore.PutImageManifest(repo, reference, desc.MediaType, manifestContent)
 		if err != nil {
 			registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference).

pkg/meta/boltdb/boltdb.go

@@ -500,6 +500,11 @@ func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.Image
 
 		imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
 		if err != nil {
+			// Skip manifests that don't have MetaDB entries (missing from storage)
+			if errors.Is(err, zerr.ErrImageMetaNotFound) {
+				continue
+			}
+
 			return imageMetaList, manifestDataList, err
 		}
 
@@ -511,6 +516,8 @@ func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.Image
 			compat.IsCompatibleManifestListMediaType(imageManifestData.MediaType) {
 			partialImageDataList, partialManifestDataList, err := getAllContainedMeta(imageBuck, imageManifestData)
 			if err != nil {
+				// getAllContainedMeta skips missing items internally, so any error returned
+				// is a real error that should be propagated
 				return imageMetaList, manifestDataList, err
 			}
 

pkg/meta/boltdb/boltdb_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/base64"
+	"errors"
 	"math"
 	"testing"
 	"time"
@@ -32,6 +33,8 @@ func (its imgTrustStore) VerifySignature(
 	return "", time.Time{}, false, nil
 }
 
+var errImageMetaBucketNotFound = errors.New("ImageMeta bucket not found")
+
 func TestWrapperErrors(t *testing.T) {
 	image := CreateDefaultImage()
 	imageMeta := image.AsImageMeta()
@@ -302,23 +305,63 @@ func TestWrapperErrors(t *testing.T) {
 				So(err, ShouldNotBeNil)
 			})
 
-			Convey("image is index, fail to get manifests", func() {
+			Convey("image is index, missing manifests are skipped gracefully", func() {
+				err := boltdbWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// Missing manifests are skipped gracefully, so GetFullImageMeta succeeds
+				// but returns an index with no manifests
+				fullImageMeta, err := boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag")
+				So(err, ShouldBeNil)
+				So(len(fullImageMeta.Manifests), ShouldEqual, 0)
+			})
+
+			Convey("image is index, corrupted manifest data returns error", func() {
+				// Create a multiarch image with multiple manifests
+				multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+				multiarchImageMeta := multiarchImage.AsImageMeta()
 				err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta)
 				So(err, ShouldBeNil)
 
-				err = boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{
-					Name: "repo",
-					Tags: map[mTypes.Tag]mTypes.Descriptor{
-						"tag": {
-							MediaType: ispec.MediaTypeImageIndex,
-							Digest:    multiarchImageMeta.Digest.String(),
-						},
-					},
+				// Store the first manifest normally
+				firstManifest := multiarchImage.Images[0]
+				firstManifestMeta := firstManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(firstManifestMeta.Digest, firstManifestMeta)
+				So(err, ShouldBeNil)
+
+				// Store the second manifest normally first, then corrupt it
+				secondManifest := multiarchImage.Images[1]
+				secondManifestMeta := secondManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(secondManifestMeta.Digest, secondManifestMeta)
+				So(err, ShouldBeNil)
+
+				secondManifestDigest := secondManifest.ManifestDescriptor.Digest
+
+				// Corrupt the data for the second manifest by storing invalid protobuf data
+				// This will cause getProtoImageMeta to return an unmarshaling error
+				// which is not ErrImageMetaNotFound, so it will propagate through getAllContainedMeta
+				corruptedData := []byte("invalid protobuf data")
+
+				// Access BoltDB directly to corrupt the data
+				err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
+					imageBuck := tx.Bucket([]byte(boltdb.ImageMetaBuck))
+					if imageBuck == nil {
+						return errImageMetaBucketNotFound
+					}
+					// Store corrupted protobuf data
+					return imageBuck.Put([]byte(secondManifestDigest.String()), corruptedData)
 				})
 				So(err, ShouldBeNil)
 
-				_, err = boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag")
+				err = boltdbWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// GetFullImageMeta should return an error due to corrupted manifest data
+				// The error from getAllContainedMeta should propagate
+				fullImageMeta, err := boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag")
 				So(err, ShouldNotBeNil)
+				// Should still return a FullImageMeta object (even with error)
+				So(fullImageMeta, ShouldNotBeNil)
 			})
 		})
 
@@ -443,6 +486,54 @@ func TestWrapperErrors(t *testing.T) {
 				_, err = boltdbWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta)
 				So(err, ShouldBeNil)
 			})
+
+			Convey("getAllContainedMeta error for index is joined and processing continues", func() {
+				// Create a multiarch image with multiple manifests
+				multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+				multiarchImageMeta := multiarchImage.AsImageMeta()
+				err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// Store the first manifest normally
+				firstManifest := multiarchImage.Images[0]
+				firstManifestMeta := firstManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(firstManifestMeta.Digest, firstManifestMeta)
+				So(err, ShouldBeNil)
+
+				// Store the second manifest normally first, then corrupt it
+				secondManifest := multiarchImage.Images[1]
+				secondManifestMeta := secondManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(secondManifestMeta.Digest, secondManifestMeta)
+				So(err, ShouldBeNil)
+
+				secondManifestDigest := secondManifest.ManifestDescriptor.Digest
+
+				// Corrupt the data for the second manifest by storing invalid protobuf data
+				// This will cause getProtoImageMeta to return an unmarshaling error
+				// which is not ErrImageMetaNotFound, so it will propagate through getAllContainedMeta
+				corruptedData := []byte("invalid protobuf data")
+
+				// Access BoltDB directly to corrupt the data
+				err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
+					imageBuck := tx.Bucket([]byte(boltdb.ImageMetaBuck))
+					if imageBuck == nil {
+						return errImageMetaBucketNotFound
+					}
+					// Store corrupted protobuf data
+					return imageBuck.Put([]byte(secondManifestDigest.String()), corruptedData)
+				})
+				So(err, ShouldBeNil)
+
+				err = boltdbWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// FilterTags should return an error due to corrupted manifest data
+				// The error from getAllContainedMeta should be joined with viewError
+				images, err := boltdbWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta)
+				So(err, ShouldNotBeNil)
+				// Should still return some images (the first valid manifest might be processed)
+				So(images, ShouldNotBeNil)
+			})
 		})
 
 		Convey("SearchRepos", func() {
@@ -470,6 +561,53 @@ func TestWrapperErrors(t *testing.T) {
 			})
 		})
 
+		Convey("GetImageMeta", func() {
+			Convey("image is index, getAllContainedMeta error returns error", func() {
+				// Create a multiarch image with multiple manifests
+				multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+				multiarchImageMeta := multiarchImage.AsImageMeta()
+				err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// Store the first manifest normally
+				firstManifest := multiarchImage.Images[0]
+				firstManifestMeta := firstManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(firstManifestMeta.Digest, firstManifestMeta)
+				So(err, ShouldBeNil)
+
+				// Store the second manifest normally first, then corrupt it
+				secondManifest := multiarchImage.Images[1]
+				secondManifestMeta := secondManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(secondManifestMeta.Digest, secondManifestMeta)
+				So(err, ShouldBeNil)
+
+				secondManifestDigest := secondManifest.ManifestDescriptor.Digest
+
+				// Corrupt the data for the second manifest by storing invalid protobuf data
+				// This will cause getProtoImageMeta to return an unmarshaling error
+				// which is not ErrImageMetaNotFound, so it will propagate through getAllContainedMeta
+				corruptedData := []byte("invalid protobuf data")
+
+				// Access BoltDB directly to corrupt the data
+				err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
+					imageBuck := tx.Bucket([]byte(boltdb.ImageMetaBuck))
+					if imageBuck == nil {
+						return errImageMetaBucketNotFound
+					}
+					// Store corrupted protobuf data
+					return imageBuck.Put([]byte(secondManifestDigest.String()), corruptedData)
+				})
+				So(err, ShouldBeNil)
+
+				// GetImageMeta should return an error due to corrupted manifest data
+				// The error from getAllContainedMeta should propagate
+				imageMeta, err := boltdbWrapper.GetImageMeta(multiarchImageMeta.Digest)
+				So(err, ShouldNotBeNil)
+				// Should still return an ImageMeta object (even with error)
+				So(imageMeta, ShouldNotBeNil)
+			})
+		})
+
 		Convey("SetRepoReference", func() {
 			Convey("getProtoRepoMeta errors", func() {
 				err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB)