From 97e22448dc6bbd8eb8709128e140c127105ba4e0 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Mon, 8 Jan 2024 15:14:39 +0100 Subject: [PATCH] chunked: validate converted images validate that the retrieved data for converted images matches the expected digest. Signed-off-by: Giuseppe Scrivano --- cmd/containers-storage/diff.go | 2 +- pkg/chunked/storage_linux.go | 59 ++++++++++++++++++++---------- pkg/chunked/storage_unsupported.go | 2 +- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/cmd/containers-storage/diff.go b/cmd/containers-storage/diff.go index ebb577d12e..24801f3b35 100644 --- a/cmd/containers-storage/diff.go +++ b/cmd/containers-storage/diff.go @@ -171,7 +171,7 @@ func applyDiffUsingStagingDirectory(flags *mflag.FlagSet, action string, m stora file: tar, } - differ, err := chunked.GetDiffer(context.Background(), m, size, metadata, &fetcher) + differ, err := chunked.GetDiffer(context.Background(), m, nil, size, metadata, &fetcher) if err != nil { return 1, err } diff --git a/pkg/chunked/storage_linux.go b/pkg/chunked/storage_linux.go index f278628e8f..126abf2171 100644 --- a/pkg/chunked/storage_linux.go +++ b/pkg/chunked/storage_linux.go @@ -86,6 +86,11 @@ type chunkedDiffer struct { // the layer are trusted and should not be validated. skipValidation bool + // blobDigest is the digest of the compressed layer. It is required + // to validate a layer when it is converted since there is no TOC referenced + // by the manifest. + blobDigest *digest.Digest + blobSize int64 storeOpts *types.StoreOptions @@ -188,7 +193,7 @@ func (f *seekableFile) GetBlobAt(chunks []ImageSourceChunk) (chan io.ReadCloser, return streams, errs, nil } -func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSourceSeekable) (*seekableFile, digest.Digest, map[string]string, error) { +func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSourceSeekable) (*seekableFile, digest.Digest, digest.Digest, map[string]string, error) { var payload io.ReadCloser var streams chan io.ReadCloser var errs chan error @@ -203,26 +208,30 @@ func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSour streams, errs, err = iss.GetBlobAt(chunksToRequest) if err != nil { - return nil, "", nil, err + return nil, "", "", nil, err } select { case p := <-streams: payload = p case err := <-errs: - return nil, "", nil, err + return nil, "", "", nil, err } if payload == nil { - return nil, "", nil, errors.New("invalid stream returned") + return nil, "", "", nil, errors.New("invalid stream returned") } - diff, err := archive.DecompressStream(payload) + digesterCompressed := digest.Canonical.Digester() + + r := io.TeeReader(payload, digesterCompressed.Hash()) + + diff, err := archive.DecompressStream(r) if err != nil { - return nil, "", nil, err + return nil, "", "", nil, err } fd, err := unix.Open(destDirectory, unix.O_TMPFILE|unix.O_RDWR|unix.O_CLOEXEC, 0o600) if err != nil { - return nil, "", nil, err + return nil, "", "", nil, err } f := os.NewFile(uintptr(fd), destDirectory) @@ -232,28 +241,30 @@ func convertTarToZstdChunked(destDirectory string, blobSize int64, iss ImageSour chunked, err := compressor.ZstdCompressor(f, newAnnotations, &level) if err != nil { f.Close() - return nil, "", nil, err + return nil, "", "", nil, err } - digester := digest.Canonical.Digester() - hash := digester.Hash() - - if _, err := io.Copy(io.MultiWriter(chunked, hash), diff); err != nil { + digesterUncompressed := digest.Canonical.Digester() + if _, err := io.Copy(io.MultiWriter(chunked, digesterUncompressed.Hash()), diff); err != nil { f.Close() - return nil, "", nil, err + return nil, "", "", nil, err } if err := chunked.Close(); err != nil { f.Close() - return nil, "", nil, err + return nil, "", "", nil, err } is := seekableFile{ file: f, } - return &is, digester.Digest(), newAnnotations, nil + + compressedDigest := digesterCompressed.Digest() + uncompressedDigest := digesterUncompressed.Digest() + + return &is, compressedDigest, uncompressedDigest, newAnnotations, nil } // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer. -func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { +func GetDiffer(ctx context.Context, store storage.Store, blobDigest *digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { storeOpts, err := types.DefaultStoreOptions() if err != nil { return nil, err @@ -273,20 +284,25 @@ func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotat return makeEstargzChunkedDiffer(ctx, store, blobSize, annotations, iss, &storeOpts) } - return makeConvertFromRawDiffer(ctx, store, blobSize, annotations, iss, &storeOpts) + return makeConvertFromRawDiffer(ctx, store, blobDigest, blobSize, annotations, iss, &storeOpts) } -func makeConvertFromRawDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) { +func makeConvertFromRawDiffer(ctx context.Context, store storage.Store, blobDigest *digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, storeOpts *types.StoreOptions) (*chunkedDiffer, error) { if !parseBooleanPullOption(storeOpts, "convert_images", false) { return nil, errors.New("convert_images not configured") } + if blobDigest == nil { + return nil, errors.New("cannot convert from raw without a blob digest") + } + layersCache, err := getLayersCache(store) if err != nil { return nil, err } return &chunkedDiffer{ + blobDigest: blobDigest, blobSize: blobSize, convertToZstdChunked: true, copyBuffer: makeCopyBuffer(), @@ -1510,7 +1526,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff stream := c.stream if c.convertToZstdChunked { - fileSource, diffID, annotations, err := convertTarToZstdChunked(dest, c.blobSize, c.stream) + fileSource, compressedDigest, diffID, annotations, err := convertTarToZstdChunked(dest, c.blobSize, c.stream) if err != nil { return graphdriver.DriverWithDifferOutput{}, err } @@ -1530,6 +1546,11 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff c.fileType = fileTypeZstdChunked c.manifest = manifest c.tarSplit = tarSplit + + if compressedDigest != *c.blobDigest { + return graphdriver.DriverWithDifferOutput{}, fmt.Errorf("invalid digest to convert: expected %q, got %q", *c.blobDigest, compressedDigest) + } + // since we retrieved the whole file and it was validated, use the diffID instead of the TOC digest. c.contentDigest = diffID c.tocOffset = tocOffset diff --git a/pkg/chunked/storage_unsupported.go b/pkg/chunked/storage_unsupported.go index 8d3fcf2ba4..394d9980c7 100644 --- a/pkg/chunked/storage_unsupported.go +++ b/pkg/chunked/storage_unsupported.go @@ -12,6 +12,6 @@ import ( ) // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer. -func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { +func GetDiffer(ctx context.Context, store storage.Store, blobDigest *digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { return nil, errors.New("format not supported on this system") }