chunked: validate converted images

validate that the retrieved data for converted images matches the
expected digest.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

cmd/containers-storage/diff.go

@@ -171,7 +171,7 @@ func applyDiffUsingStagingDirectory(flags *mflag.FlagSet, action string, m stora
 		file: tar,
 	}
 
-	differ, err := chunked.GetDiffer(context.Background(), m, size, metadata, &fetcher)
+	differ, err := chunked.GetDiffer(context.Background(), m, nil, size, metadata, &fetcher)
 	if err != nil {
 		return 1, err
 	}

pkg/chunked/storage_linux.go

@@ -86,6 +86,11 @@ type chunkedDiffer struct {
 	// the layer are trusted and should not be validated.
 	skipValidation bool
 
+	// blobDigest is the digest of the compressed layer.  It is required
+	// to validate a layer when it is converted since there is no TOC referenced
+	// by the manifest.
+	blobDigest *digest.Digest
+
 	blobSize int64
 
 	storeOpts *types.StoreOptions
@@ -188,7 +193,7 @@ func (f *seekableFile) GetBlobAt(chunks []ImageSourceChunk) (chan io.ReadCloser,
 	return streams, errs, nil
 }
 
-func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSourceSeekable) (*seekableFile, digest.Digest, map[string]string, error) {
+func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSourceSeekable) (*seekableFile, digest.Digest, digest.Digest, map[string]string, error) {
 	var payload io.ReadCloser
 	var streams chan io.ReadCloser
 	var errs chan error
@@ -203,26 +208,30 @@ func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSour
 
 	streams, errs, err = iss.GetBlobAt(chunksToRequest)
 	if err != nil {
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 	select {
 	case p := <-streams:
 		payload = p
 	case err := <-errs:
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 	if payload == nil {
-		return nil, "", nil, errors.New("invalid stream returned")
+		return nil, "", "", nil, errors.New("invalid stream returned")
 	}
 
-	diff, err := archive.DecompressStream(payload)
+	digesterCompressed := digest.Canonical.Digester()
+
+	r := io.TeeReader(payload, digesterCompressed.Hash())
+
+	diff, err := archive.DecompressStream(r)
 	if err != nil {
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 
 	fd, err := unix.Open(destDirectory, unix.O_TMPFILE|unix.O_RDWR|unix.O_CLOEXEC, 0o600)
 	if err != nil {
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 
 	f := os.NewFile(uintptr(fd), destDirectory)
@@ -232,28 +241,30 @@ func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSour
 	chunked, err := compressor.ZstdCompressor(f, newAnnotations, &level)
 	if err != nil {
 		f.Close()
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 
-	digester := digest.Canonical.Digester()
-	hash := digester.Hash()
-
-	if _, err := io.Copy(io.MultiWriter(chunked, hash), diff); err != nil {
+	digesterUncompressed := digest.Canonical.Digester()
+	if _, err := io.Copy(io.MultiWriter(chunked, digesterUncompressed.Hash()), diff); err != nil {
 		f.Close()
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 	if err := chunked.Close(); err != nil {
 		f.Close()
-		return nil, "", nil, err
+		return nil, "", "", nil, err
 	}
 	is := seekableFile{
 		file: f,
 	}
-	return &is, digester.Digest(), newAnnotations, nil
+
+	compressedDigest := digesterCompressed.Digest()
+	uncompressedDigest := digesterUncompressed.Digest()
+
+	return &is, compressedDigest, uncompressedDigest, newAnnotations, nil
 }
 
 // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
-func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
+func GetDiffer(ctx context.Context, store storage.Store, blobDigest *digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
 	storeOpts, err := types.DefaultStoreOptions()
 	if err != nil {
 		return nil, err
@@ -273,20 +284,25 @@ func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotat
 		return makeEstargzChunkedDiffer(ctx, store, blobSize, annotations, iss, &storeOpts)
 	}
 
-	return makeConvertFromRawDiffer(ctx, store, blobSize, annotations, iss, &storeOpts)
+	return makeConvertFromRawDiffer(ctx, store, blobDigest, blobSize, annotations, iss, &storeOpts)
 }
 
-func makeConvertFromRawDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) {
+func makeConvertFromRawDiffer(ctx context.Context, store storage.Store, blobDigest *digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) {
 	if !parseBooleanPullOption(storeOpts, "convert_images", false) {
 		return nil, errors.New("convert_images not configured")
 	}
 
+	if blobDigest == nil {
+		return nil, errors.New("cannot convert from raw without a blob digest")
+	}
+
 	layersCache, err := getLayersCache(store)
 	if err != nil {
 		return nil, err
 	}
 
 	return &chunkedDiffer{
+		blobDigest:           blobDigest,
 		blobSize:             blobSize,
 		convertToZstdChunked: true,
 		copyBuffer:           makeCopyBuffer(),
@@ -1510,7 +1526,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	stream := c.stream
 
 	if c.convertToZstdChunked {
-		fileSource, diffID, annotations, err := convertTarToZstdChunked(dest, c.blobSize, c.stream)
+		fileSource, compressedDigest, diffID, annotations, err := convertTarToZstdChunked(dest, c.blobSize, c.stream)
 		if err != nil {
 			return graphdriver.DriverWithDifferOutput{}, err
 		}
@@ -1530,6 +1546,11 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 		c.fileType = fileTypeZstdChunked
 		c.manifest = manifest
 		c.tarSplit = tarSplit
+
+		if compressedDigest != *c.blobDigest {
+			return graphdriver.DriverWithDifferOutput{}, fmt.Errorf("invalid digest to convert: expected %q, got %q", *c.blobDigest, compressedDigest)
+		}
+
 		// since we retrieved the whole file and it was validated, use the diffID instead of the TOC digest.
 		c.contentDigest = diffID
 		c.tocOffset = tocOffset

pkg/chunked/storage_unsupported.go

@@ -12,6 +12,6 @@ import (
 )
 
 // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
-func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
+func GetDiffer(ctx context.Context, store storage.Store, blobDigest *digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
 	return nil, errors.New("format not supported on this system")
 }