Support for importing verified block CIM layers

ext4/tar2ext4/tar2ext4.go

@@ -352,3 +352,18 @@ func ConvertToVhd(w io.WriteSeeker) error {
 	}
 	return binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size))
 }
+
+// A convenience wrapper for ConverToVhd, instead of asking the caller to open the file and pass an io.WriteSeeker, this
+// takes in a file path and appends the VHD footer to that file.
+func ConvertFileToVhd(filePath string) error {
+	f, err := os.OpenFile(filePath, os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open file `%s` : %w", filePath, err)
+	}
+	defer f.Close()
+
+	if err := ConvertToVhd(f); err != nil {
+		return fmt.Errorf("failed to append VHD footer: %w", err)
+	}
+	return nil
+}

internal/wclayer/cim/block_cim_writer.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/Microsoft/go-winio"
 	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/wclayer"
 	"github.com/Microsoft/hcsshim/pkg/cimfs"
 )
 
@@ -26,11 +27,10 @@ type BlockCIMLayerWriter struct {
 
 var _ CIMLayerWriter = &BlockCIMLayerWriter{}
 
-// NewBlockCIMLayerWriter writes the layer files in the block CIM format.
-func NewBlockCIMLayerWriter(ctx context.Context, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM) (_ *BlockCIMLayerWriter, err error) {
-	if !cimfs.IsBlockCimSupported() {
-		return nil, fmt.Errorf("BlockCIM not supported on this build")
-	} else if layer.Type != cimfs.BlockCIMTypeSingleFile {
+// NewBlockCIMLayerWriterWithOpts returns a writer for writing image layers in the block CIM format. The writer's behavior can be
+// controlled with the supports opts.
+func NewBlockCIMLayerWriterWithOpts(ctx context.Context, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM, opts ...cimfs.BlockCIMOpt) (_ *BlockCIMLayerWriter, err error) {
+	if layer.Type != cimfs.BlockCIMTypeSingleFile {
 		// we only support writing single file CIMs for now because in layer
 		// writing process we still need to write some files (registry hives)
 		// outside the CIM. We currently use the parent directory of the CIM (i.e
@@ -50,7 +50,10 @@ func NewBlockCIMLayerWriter(ctx context.Context, layer *cimfs.BlockCIM, parentLa
 		parentLayerPaths = append(parentLayerPaths, filepath.Dir(pl.BlockPath))
 	}
 
-	cim, err := cimfs.CreateBlockCIM(layer.BlockPath, layer.CimName, layer.Type)
+	// We always want to write layers with consistent flag
+	bcimOpts := append([]cimfs.BlockCIMOpt{cimfs.WithConsistentCIM()}, opts...)
+
+	cim, err := cimfs.CreateBlockCIMWithOptions(ctx, layer, bcimOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error in creating a new cim: %w", err)
 	}
@@ -86,9 +89,25 @@ func NewBlockCIMLayerWriter(ctx context.Context, layer *cimfs.BlockCIM, parentLa
 	}, nil
 }
 
+// NewBlockCIMLayerWriter writes the layer files in the block CIM format.
+func NewBlockCIMLayerWriter(ctx context.Context, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM) (_ *BlockCIMLayerWriter, err error) {
+	return NewBlockCIMLayerWriterWithOpts(ctx, layer, parentLayers)
+}
+
 // Add adds a file to the layer with given metadata.
 func (cw *BlockCIMLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo, fileSize int64, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error {
 	cw.addedFiles[name] = struct{}{}
+	if name == wclayer.UtilityVMPath && len(cw.parentLayers) > 0 {
+		// If there are UtilityVM files in non base layers, we will have to merge
+		// those files with the parent layer UtilityVM files - either during image
+		// pull or at runtime (i.e when starting the UVM). In order to merge at image pull time, we will have
+		// to read parent layer block CIMs and copy all the UtilityVM files from
+		// those CIMs into this block CIM one by one i.e effectively merge all
+		// parent layer UtilityVM files in this layer. Or we will need to be able
+		// to boot the UtilityVM with merged block CIMs. None of these options are
+		// implemented yet so error out if we see that.
+		return fmt.Errorf("UtilityVM files in non base layers is not supported for block CIMs")
+	}
 	return cw.cimLayerWriter.Add(name, fileInfo, fileSize, securityDescriptor, extendedAttributes, reparseData)
 }
 

internal/wclayer/cim/common.go

@@ -17,6 +17,7 @@ import (
 var (
 	ErrBlockCIMWriterNotSupported = fmt.Errorf("writing block device CIM isn't supported")
 	ErrBlockCIMParentTypeMismatch = fmt.Errorf("parent layer block CIM type doesn't match with extraction layer")
+	ErrBlockCIMIntegrityMismatch  = fmt.Errorf("verified CIMs can not be mixed with non verified CIMs")
 )
 
 type hive struct {

pkg/extractuvm/bcim.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/Microsoft/hcsshim/ext4/tar2ext4"
 	"github.com/Microsoft/hcsshim/pkg/cimfs"
 )
 
@@ -22,7 +23,9 @@ const (
 )
 
 func MakeUtilityVMCIMFromTar(ctx context.Context, tarPath, destPath string) (_ *cimfs.BlockCIM, err error) {
-	slog.InfoContext(ctx, "Extracting UtilityVM files from tar", "tarPath", tarPath, "destPath", destPath)
+	slog.InfoContext(ctx, "Extracting UtilityVM files from tar",
+		"tarPath", tarPath,
+		"destPath", destPath)
 
 	tarFile, err := os.Open(tarPath)
 	if err != nil {
@@ -41,20 +44,32 @@ func MakeUtilityVMCIMFromTar(ctx context.Context, tarPath, destPath string) (_ *
 		CimName:   "boot.cim",
 	}
 
-	w, err := newUVMCIMWriter(uvmCIM)
+	w, err := newUVMCIMWriter(ctx, uvmCIM)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create block CIM writer: %w", err)
 	}
 	defer func() {
-		cErr := w.Close(ctx)
-		if err == nil {
-			err = cErr
+		// only attempt to close the writer in case of errors, in success case we close it anyway
+		if err != nil {
+			if closeErr := w.Close(ctx); closeErr != nil {
+				slog.ErrorContext(ctx, "failed to close CIM writer", "error", closeErr)
+			}
 		}
 	}()
 
 	if err = extractUtilityVMFilesFromTar(ctx, tarFile, w); err != nil {
 		return nil, fmt.Errorf("failed to extract UVM layer: %w", err)
 	}
+
+	// MUST close the writer before appending VHD footer
+	if err = w.Close(ctx); err != nil {
+		return nil, fmt.Errorf("failed to close CIM writer: %w", err)
+	}
+
+	// We always want to append the VHD footer to the UVM CIM
+	if err := tar2ext4.ConvertFileToVhd(uvmCIM.BlockPath); err != nil {
+		return nil, fmt.Errorf("failed to append VHD footer: %w", err)
+	}
 	return uvmCIM, nil
 }
 

pkg/extractuvm/bcim_test.go

@@ -153,7 +153,7 @@ func extractAndVerifyTarToCIM(t *testing.T, tarContents []testFile, contentsToVe
 }
 
 func TestTarUtilityVMExtract(t *testing.T) {
-	if !cimfs.IsBlockCimSupported() {
+	if !cimfs.IsVerifiedCimSupported() {
 		t.Skip("block CIMs are not supported on this build")
 	}
 

pkg/extractuvm/uvm_cim_writer.go

@@ -25,8 +25,12 @@ type uvmCIMWriter struct {
 	filesAdded map[string]struct{}
 }
 
-func newUVMCIMWriter(l *cimfs.BlockCIM) (*uvmCIMWriter, error) {
-	cim, err := cimfs.CreateBlockCIM(l.BlockPath, l.CimName, l.Type)
+func newUVMCIMWriter(ctx context.Context, l *cimfs.BlockCIM) (*uvmCIMWriter, error) {
+	var cim *cimfs.CimFsWriter
+	var err error
+
+	// We always want to create the UVM CIM with data integrity
+	cim, err = cimfs.CreateBlockCIMWithOptions(ctx, l, cimfs.WithDataIntegrity())
 	if err != nil {
 		return nil, fmt.Errorf("error in creating a new cim: %w", err)
 	}

pkg/ociwclayer/cim/import.go

@@ -7,6 +7,7 @@ import (
 	"archive/tar"
 	"bufio"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -16,6 +17,7 @@ import (
 	"strings"
 
 	"github.com/Microsoft/go-winio/backuptar"
+	"github.com/Microsoft/hcsshim/ext4/tar2ext4"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/wclayer/cim"
 	"github.com/Microsoft/hcsshim/pkg/cimfs"
@@ -40,7 +42,7 @@ func ImportCimLayerFromTar(ctx context.Context, r io.Reader, layerPath, cimPath
 		"parent layer CIM paths": strings.Join(parentLayerCimPaths, ", "),
 	}).Debug("Importing cim layer from tar")
 
-	err = os.MkdirAll(layerPath, 0)
+	err = os.MkdirAll(layerPath, 0755)
 	if err != nil {
 		return 0, err
 	}
@@ -61,20 +63,86 @@ func ImportCimLayerFromTar(ctx context.Context, r io.Reader, layerPath, cimPath
 	return n, nil
 }
 
-// ImportSingleFileCimLayerFromTar reads a layer from an OCI layer tar stream and extracts
-// it into the SingleFileCIM format.
-func ImportSingleFileCimLayerFromTar(ctx context.Context, r io.Reader, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM) (_ int64, err error) {
-	log.G(ctx).WithFields(logrus.Fields{
-		"layer":         layer,
-		"parent layers": fmt.Sprintf("%v", parentLayers),
-	}).Debug("Importing single file cim layer from tar")
+type blockCIMLayerImportConfig struct {
+	// import layers with integrity enabled CIMs
+	dataIntegrity bool
+	// parent layers
+	parentLayers []*cimfs.BlockCIM
+	// append VHD footer to the import CIMs
+	appendVHDFooter bool
+}
+
+// BlockCIMOpt is a function type for configuring block CIM creation options
+type BlockCIMLayerImportOpt func(*blockCIMLayerImportConfig) error
+
+func WithLayerIntegrity() BlockCIMLayerImportOpt {
+	return func(opts *blockCIMLayerImportConfig) error {
+		opts.dataIntegrity = true
+		return nil
+	}
+}
 
-	err = os.MkdirAll(filepath.Dir(layer.BlockPath), 0)
+func WithVHDFooter() BlockCIMLayerImportOpt {
+	return func(opts *blockCIMLayerImportConfig) error {
+		opts.appendVHDFooter = true
+		return nil
+	}
+}
+
+func WithParentLayers(parentLayers []*cimfs.BlockCIM) BlockCIMLayerImportOpt {
+	return func(opts *blockCIMLayerImportConfig) error {
+		opts.parentLayers = parentLayers
+		return nil
+	}
+}
+
+func writeIntegrityChecksumInfoFile(ctx context.Context, blockPath string) error {
+	log.G(ctx).Debugf("writing integrity checksum file for block CIM `%s`", blockPath)
+	// for convenience write a file that has the base64 encoded root digest of the generated verified CIM.
+	// this same base64 string can be used in the confidential policy.
+	digest, err := cimfs.GetVerificationInfo(blockPath)
+	if err != nil {
+		return fmt.Errorf("failed to query verified info of the CIM layer: %w", err)
+	}
+
+	digestFile, err := os.Create(filepath.Join(filepath.Dir(blockPath), "integrity_checksum"))
+	if err != nil {
+		return fmt.Errorf("failed to create verification info file: %w", err)
+	}
+	defer digestFile.Close()
+
+	digestStr := base64.URLEncoding.EncodeToString(digest)
+	if wn, err := digestFile.WriteString(digestStr); err != nil {
+		return fmt.Errorf("failed to write verification info: %w", err)
+	} else if wn != len(digestStr) {
+		return fmt.Errorf("incomplete write of verification info: %w", err)
+	}
+	return nil
+}
+
+func ImportBlockCIMLayerWithOpts(ctx context.Context, r io.Reader, layer *cimfs.BlockCIM, opts ...BlockCIMLayerImportOpt) (_ int64, err error) {
+	log.G(ctx).WithField("layer", layer).Debug("Importing block CIM layer from tar")
+
+	err = os.MkdirAll(filepath.Dir(layer.BlockPath), 0755)
 	if err != nil {
 		return 0, err
 	}
 
-	w, err := cim.NewBlockCIMLayerWriter(ctx, layer, parentLayers)
+	config := &blockCIMLayerImportConfig{}
+	for _, opt := range opts {
+		if err := opt(config); err != nil {
+			return 0, fmt.Errorf("block CIM import config option failure: %w", err)
+		}
+	}
+
+	log.G(ctx).WithField("config", *config).Debug("layer import config")
+
+	bcimWriterOpts := []cimfs.BlockCIMOpt{}
+	if config.dataIntegrity {
+		bcimWriterOpts = append(bcimWriterOpts, cimfs.WithDataIntegrity())
+	}
+
+	w, err := cim.NewBlockCIMLayerWriterWithOpts(ctx, layer, config.parentLayers, bcimWriterOpts...)
 	if err != nil {
 		return 0, err
 	}
@@ -87,9 +155,29 @@ func ImportSingleFileCimLayerFromTar(ctx context.Context, r io.Reader, layer *ci
 	if cerr != nil {
 		return 0, cerr
 	}
+
+	if config.appendVHDFooter {
+		log.G(ctx).Debugf("appending VHD footer to block CIM at `%s`", layer.BlockPath)
+		if err = tar2ext4.ConvertFileToVhd(layer.BlockPath); err != nil {
+			return 0, fmt.Errorf("append VHD footer to block CIM: %w", err)
+		}
+	}
+
+	if config.dataIntegrity {
+		if err = writeIntegrityChecksumInfoFile(ctx, layer.BlockPath); err != nil {
+			return 0, err
+		}
+	}
+
 	return n, nil
 }
 
+// ImportSingleFileCimLayerFromTar reads a layer from an OCI layer tar stream and extracts
+// it into the SingleFileCIM format.
+func ImportSingleFileCimLayerFromTar(ctx context.Context, r io.Reader, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM) (_ int64, err error) {
+	return ImportBlockCIMLayerWithOpts(ctx, r, layer, WithParentLayers(parentLayers))
+}
+
 func writeCimLayerFromTar(ctx context.Context, r io.Reader, w cim.CIMLayerWriter) (int64, error) {
 	tr := tar.NewReader(r)
 	buf := bufio.NewWriter(w)
@@ -197,3 +285,78 @@ func writeCimLayerFromTar(ctx context.Context, r io.Reader, w cim.CIMLayerWriter
 	}
 	return size, nil
 }
+
+// MergeBlockCIMLayersWithOpts create a new block CIM at mergedCIM.BlockPath and then
+// creates a new CIM inside that block CIM that merges all the contents of the provided
+// sourceCIMs. Note that this is only a metadata merge, so when this merged CIM is being
+// mounted, all the sourceCIMs must be provided too and they MUST be provided in the same
+// order. Expected order of sourceCIMs is that the base layer should be at the last index
+// and the topmost layer should be at 0'th index.  Merge operation can take a long time in
+// certain situations, this function respects context deadlines in such cases.  This
+// function is NOT thread safe, it is caller's responsibility to handle thread safety.
+func MergeBlockCIMLayersWithOpts(ctx context.Context, sourceCIMs []*cimfs.BlockCIM, mergedCIM *cimfs.BlockCIM, opts ...BlockCIMLayerImportOpt) (retErr error) {
+	log.G(ctx).WithFields(logrus.Fields{
+		"source CIMs": sourceCIMs,
+		"merged CIM":  mergedCIM,
+	}).Debug("Merging block CIM layers")
+
+	// check if a merged CIM already exists
+	_, err := os.Stat(mergedCIM.BlockPath)
+	if err == nil {
+		return os.ErrExist
+	}
+
+	// Apply configuration options
+	config := &blockCIMLayerImportConfig{}
+	for _, opt := range opts {
+		if err := opt(config); err != nil {
+			return fmt.Errorf("apply merge option: %w", err)
+		}
+	}
+
+	// Prepare options for the underlying cimfs.MergeBlockCIMsWithOpts call
+	cimfsOpts := []cimfs.BlockCIMOpt{cimfs.WithConsistentCIM()}
+	if config.dataIntegrity {
+		cimfsOpts = append(cimfsOpts, cimfs.WithDataIntegrity())
+	}
+
+	// Ensure the directory for the merged CIM exists
+	if err := os.MkdirAll(filepath.Dir(mergedCIM.BlockPath), 0755); err != nil {
+		return fmt.Errorf("create directory for merged CIM: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			if rmErr := os.Remove(mergedCIM.BlockPath); rmErr != nil {
+				log.G(ctx).WithError(retErr).Warnf("error in cleanup on failure: %s", rmErr)
+			}
+		}
+	}()
+
+	// Run the merge operation in a goroutine to handle context cancellation
+	// The merge operation can take a long time, so we need to respect context deadlines
+	errCh := make(chan error, 1)
+	go func() {
+		defer close(errCh)
+		err := cimfs.MergeBlockCIMsWithOpts(ctx, mergedCIM, sourceCIMs, cimfsOpts...)
+		errCh <- err
+	}()
+
+	// Wait for either the merge to complete or context to be cancelled
+	select {
+	case <-ctx.Done():
+		err = ctx.Err()
+	case err = <-errCh:
+	}
+	if err != nil {
+		return fmt.Errorf("merge block CIMs failed: %w", err)
+	}
+
+	// Handle VHD footer if requested
+	if config.appendVHDFooter {
+		log.G(ctx).Debugf("appending VHD footer to block CIM at `%s`", mergedCIM.BlockPath)
+		if err = tar2ext4.ConvertFileToVhd(mergedCIM.BlockPath); err != nil {
+			return fmt.Errorf("append VHD footer to block CIM: %w", err)
+		}
+	}
+	return nil
+}