fix: gracefully handle manifests missing from storage (prepare for sparse indexes)

pkg/extensions/search/cve/scan_test.go

@@ -384,6 +384,11 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo
 					return false, err
 				}
 
+				// If all manifests are missing (e.g., from an index), Manifests will be empty
+				if len(manifestData.Manifests) == 0 {
+					return false, nil
+				}
+
 				for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers {
 					switch imageLayer.MediaType {
 					case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer):

pkg/extensions/search/search_test.go

@@ -7406,6 +7406,98 @@ type repoRef struct {
 	Tag  string
 }
 
+func TestSearchWithMissingManifest(t *testing.T) {
+	Convey("Search with missing manifest", t, func() {
+		dir := t.TempDir()
+
+		// 1. Write the image to the disk
+		log := log.NewTestLogger()
+		storeCtlr := ociutils.GetDefaultStoreController(dir, log)
+
+		// Create a multiarch image with exactly 2 manifests
+		multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+
+		// Write the multiarch image to filesystem
+		err := WriteMultiArchImageToFileSystem(multiarchImage, "testrepo", "latest", storeCtlr)
+		So(err, ShouldBeNil)
+
+		// Get the image store to access index and manifests
+		imageStore := storeCtlr.GetDefaultImageStore()
+
+		// Get the index content to find all manifest digests
+		indexBlob, err := imageStore.GetIndexContent("testrepo")
+		So(err, ShouldBeNil)
+
+		var indexContent ispec.Index
+		err = json.Unmarshal(indexBlob, &indexContent)
+		So(err, ShouldBeNil)
+
+		So(len(indexContent.Manifests), ShouldBeGreaterThanOrEqualTo, 2)
+
+		// Get the first manifest digest to delete
+		firstManifestDigest := indexContent.Manifests[0].Digest
+
+		// Get the second manifest digest (should remain valid)
+		secondManifestDigest := indexContent.Manifests[1].Digest
+
+		// 2. Delete the manifest from the disk
+		manifestBlobPath := path.Join(dir, "testrepo", "blobs", "sha256", firstManifestDigest.Encoded())
+		err = os.Remove(manifestBlobPath)
+		So(err, ShouldBeNil)
+
+		// 3. Start the controller (MetaDB parsing would be done in the background)
+		port := GetFreePort()
+		baseURL := GetBaseURL(port)
+		conf := config.New()
+		conf.HTTP.Port = port
+		conf.Storage.RootDirectory = dir
+		defaultVal := true
+		conf.Extensions = &extconf.ExtensionConfig{
+			Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
+		}
+
+		conf.Extensions.Search.CVE = nil
+
+		ctlr := api.NewController(conf)
+
+		ctlrManager := NewControllerManager(ctlr)
+		ctlrManager.StartAndWait(port)
+		defer ctlrManager.StopServer()
+
+		// Search for the repository
+		query := `
+		{
+			GlobalSearch(query:"testrepo:latest"){
+				Images {
+					RepoName Tag
+					Manifests {
+						Digest
+					}
+				}
+			}
+		}`
+
+		resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
+		So(resp, ShouldNotBeNil)
+		So(err, ShouldBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+		responseStruct := &zcommon.GlobalSearchResultResp{}
+		err = json.Unmarshal(resp.Body(), responseStruct)
+		So(err, ShouldBeNil)
+
+		// Verify we found the image
+		So(responseStruct.GlobalSearchResult.GlobalSearch.Images, ShouldNotBeEmpty)
+		foundImage := responseStruct.GlobalSearchResult.GlobalSearch.Images[0]
+		So(foundImage.RepoName, ShouldEqual, "testrepo")
+		So(foundImage.Tag, ShouldEqual, "latest")
+
+		// Verify only the valid manifest is found in search results (missing one was skipped by ParseStorage)
+		So(len(foundImage.Manifests), ShouldEqual, 1)
+		So(foundImage.Manifests[0].Digest, ShouldEqual, secondManifestDigest.String())
+	})
+}
+
 func deleteUsedImages(repoTags []repoRef, baseURL string) {
 	for _, image := range repoTags {
 		status, err := DeleteImage(image.Repo, image.Tag, baseURL)

pkg/extensions/sync/destination.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/distribution/distribution/v3/registry/storage/driver"
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/regclient/regclient/types/mediatype"
@@ -227,11 +228,27 @@ func (registry *DestinationRegistry) copyManifest(repo string, desc ispec.Descri
 			return err
 		}
 
+		var firstMissingErr error
+
 		for _, manifest := range indexManifest.Manifests {
 			reference := GetDescriptorReference(manifest)
 
 			manifestBuf, err := tempImageStore.GetBlobContent(repo, manifest.Digest)
 			if err != nil {
+				// Handle missing manifest blobs gracefully - log warning and continue with other manifests
+				var pathNotFoundErr driver.PathNotFoundError
+				if errors.Is(err, zerr.ErrBlobNotFound) || errors.As(err, &pathNotFoundErr) {
+					if firstMissingErr == nil {
+						firstMissingErr = err
+					}
+
+					registry.log.Warn().Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).
+						Str("digest", manifest.Digest.String()).
+						Msg("skipping missing manifest blob in image index, continuing sync with other manifests")
+
+					continue
+				}
+
 				registry.log.Error().Str("errorType", common.TypeOf(err)).
 					Err(err).Str("dir", path.Join(tempImageStore.RootDir(), repo)).Str("digest", manifest.Digest.String()).
 					Msg("failed find manifest which is part of an image index")
@@ -254,6 +271,11 @@ func (registry *DestinationRegistry) copyManifest(repo string, desc ispec.Descri
 			}
 		}
 
+		// Return error if we encountered any missing manifests
+		if firstMissingErr != nil {
+			return firstMissingErr
+		}
+
 		_, _, err := imageStore.PutImageManifest(repo, reference, desc.MediaType, manifestContent)
 		if err != nil {
 			registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference).

pkg/meta/boltdb/boltdb.go

@@ -500,6 +500,11 @@ func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.Image
 
 		imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
 		if err != nil {
+			// Skip manifests that don't have MetaDB entries (missing from storage)
+			if errors.Is(err, zerr.ErrImageMetaNotFound) {
+				continue
+			}
+
 			return imageMetaList, manifestDataList, err
 		}
 
@@ -511,6 +516,8 @@ func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.Image
 			compat.IsCompatibleManifestListMediaType(imageManifestData.MediaType) {
 			partialImageDataList, partialManifestDataList, err := getAllContainedMeta(imageBuck, imageManifestData)
 			if err != nil {
+				// getAllContainedMeta skips missing items internally, so any error returned
+				// is a real error that should be propagated
 				return imageMetaList, manifestDataList, err
 			}
 

pkg/meta/boltdb/boltdb_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/base64"
+	"errors"
 	"math"
 	"testing"
 	"time"
@@ -32,6 +33,8 @@ func (its imgTrustStore) VerifySignature(
 	return "", time.Time{}, false, nil
 }
 
+var errImageMetaBucketNotFound = errors.New("ImageMeta bucket not found")
+
 func TestWrapperErrors(t *testing.T) {
 	image := CreateDefaultImage()
 	imageMeta := image.AsImageMeta()
@@ -302,23 +305,63 @@ func TestWrapperErrors(t *testing.T) {
 				So(err, ShouldNotBeNil)
 			})
 
-			Convey("image is index, fail to get manifests", func() {
+			Convey("image is index, missing manifests are skipped gracefully", func() {
+				err := boltdbWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// Missing manifests are skipped gracefully, so GetFullImageMeta succeeds
+				// but returns an index with no manifests
+				fullImageMeta, err := boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag")
+				So(err, ShouldBeNil)
+				So(len(fullImageMeta.Manifests), ShouldEqual, 0)
+			})
+
+			Convey("image is index, corrupted manifest data returns error", func() {
+				// Create a multiarch image with multiple manifests
+				multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+				multiarchImageMeta := multiarchImage.AsImageMeta()
 				err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta)
 				So(err, ShouldBeNil)
 
-				err = boltdbWrapper.SetRepoMeta("repo", mTypes.RepoMeta{
-					Name: "repo",
-					Tags: map[mTypes.Tag]mTypes.Descriptor{
-						"tag": {
-							MediaType: ispec.MediaTypeImageIndex,
-							Digest:    multiarchImageMeta.Digest.String(),
-						},
-					},
+				// Store the first manifest normally
+				firstManifest := multiarchImage.Images[0]
+				firstManifestMeta := firstManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(firstManifestMeta.Digest, firstManifestMeta)
+				So(err, ShouldBeNil)
+
+				// Store the second manifest normally first, then corrupt it
+				secondManifest := multiarchImage.Images[1]
+				secondManifestMeta := secondManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(secondManifestMeta.Digest, secondManifestMeta)
+				So(err, ShouldBeNil)
+
+				secondManifestDigest := secondManifest.ManifestDescriptor.Digest
+
+				// Corrupt the data for the second manifest by storing invalid protobuf data
+				// This will cause getProtoImageMeta to return an unmarshaling error
+				// which is not ErrImageMetaNotFound, so it will propagate through getAllContainedMeta
+				corruptedData := []byte("invalid protobuf data")
+
+				// Access BoltDB directly to corrupt the data
+				err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
+					imageBuck := tx.Bucket([]byte(boltdb.ImageMetaBuck))
+					if imageBuck == nil {
+						return errImageMetaBucketNotFound
+					}
+					// Store corrupted protobuf data
+					return imageBuck.Put([]byte(secondManifestDigest.String()), corruptedData)
 				})
 				So(err, ShouldBeNil)
 
-				_, err = boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag")
+				err = boltdbWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// GetFullImageMeta should return an error due to corrupted manifest data
+				// The error from getAllContainedMeta should propagate
+				fullImageMeta, err := boltdbWrapper.GetFullImageMeta(ctx, "repo", "tag")
 				So(err, ShouldNotBeNil)
+				// Should still return a FullImageMeta object (even with error)
+				So(fullImageMeta, ShouldNotBeNil)
 			})
 		})
 
@@ -443,6 +486,54 @@ func TestWrapperErrors(t *testing.T) {
 				_, err = boltdbWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta)
 				So(err, ShouldBeNil)
 			})
+
+			Convey("getAllContainedMeta error for index is joined and processing continues", func() {
+				// Create a multiarch image with multiple manifests
+				multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+				multiarchImageMeta := multiarchImage.AsImageMeta()
+				err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// Store the first manifest normally
+				firstManifest := multiarchImage.Images[0]
+				firstManifestMeta := firstManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(firstManifestMeta.Digest, firstManifestMeta)
+				So(err, ShouldBeNil)
+
+				// Store the second manifest normally first, then corrupt it
+				secondManifest := multiarchImage.Images[1]
+				secondManifestMeta := secondManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(secondManifestMeta.Digest, secondManifestMeta)
+				So(err, ShouldBeNil)
+
+				secondManifestDigest := secondManifest.ManifestDescriptor.Digest
+
+				// Corrupt the data for the second manifest by storing invalid protobuf data
+				// This will cause getProtoImageMeta to return an unmarshaling error
+				// which is not ErrImageMetaNotFound, so it will propagate through getAllContainedMeta
+				corruptedData := []byte("invalid protobuf data")
+
+				// Access BoltDB directly to corrupt the data
+				err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
+					imageBuck := tx.Bucket([]byte(boltdb.ImageMetaBuck))
+					if imageBuck == nil {
+						return errImageMetaBucketNotFound
+					}
+					// Store corrupted protobuf data
+					return imageBuck.Put([]byte(secondManifestDigest.String()), corruptedData)
+				})
+				So(err, ShouldBeNil)
+
+				err = boltdbWrapper.SetRepoReference(ctx, "repo", "tag", multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// FilterTags should return an error due to corrupted manifest data
+				// The error from getAllContainedMeta should be joined with viewError
+				images, err := boltdbWrapper.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta)
+				So(err, ShouldNotBeNil)
+				// Should still return some images (the first valid manifest might be processed)
+				So(images, ShouldNotBeNil)
+			})
 		})
 
 		Convey("SearchRepos", func() {
@@ -470,6 +561,53 @@ func TestWrapperErrors(t *testing.T) {
 			})
 		})
 
+		Convey("GetImageMeta", func() {
+			Convey("image is index, getAllContainedMeta error returns error", func() {
+				// Create a multiarch image with multiple manifests
+				multiarchImage := CreateMultiarchWith().RandomImages(2).Build()
+				multiarchImageMeta := multiarchImage.AsImageMeta()
+				err := boltdbWrapper.SetImageMeta(multiarchImageMeta.Digest, multiarchImageMeta)
+				So(err, ShouldBeNil)
+
+				// Store the first manifest normally
+				firstManifest := multiarchImage.Images[0]
+				firstManifestMeta := firstManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(firstManifestMeta.Digest, firstManifestMeta)
+				So(err, ShouldBeNil)
+
+				// Store the second manifest normally first, then corrupt it
+				secondManifest := multiarchImage.Images[1]
+				secondManifestMeta := secondManifest.AsImageMeta()
+				err = boltdbWrapper.SetImageMeta(secondManifestMeta.Digest, secondManifestMeta)
+				So(err, ShouldBeNil)
+
+				secondManifestDigest := secondManifest.ManifestDescriptor.Digest
+
+				// Corrupt the data for the second manifest by storing invalid protobuf data
+				// This will cause getProtoImageMeta to return an unmarshaling error
+				// which is not ErrImageMetaNotFound, so it will propagate through getAllContainedMeta
+				corruptedData := []byte("invalid protobuf data")
+
+				// Access BoltDB directly to corrupt the data
+				err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
+					imageBuck := tx.Bucket([]byte(boltdb.ImageMetaBuck))
+					if imageBuck == nil {
+						return errImageMetaBucketNotFound
+					}
+					// Store corrupted protobuf data
+					return imageBuck.Put([]byte(secondManifestDigest.String()), corruptedData)
+				})
+				So(err, ShouldBeNil)
+
+				// GetImageMeta should return an error due to corrupted manifest data
+				// The error from getAllContainedMeta should propagate
+				imageMeta, err := boltdbWrapper.GetImageMeta(multiarchImageMeta.Digest)
+				So(err, ShouldNotBeNil)
+				// Should still return an ImageMeta object (even with error)
+				So(imageMeta, ShouldNotBeNil)
+			})
+		})
+
 		Convey("SetRepoReference", func() {
 			Convey("getProtoRepoMeta errors", func() {
 				err := setRepoMeta("repo", badProtoBlob, boltdbWrapper.DB)