Use CimFS layers for Process isolated WCOW (#1971)

Signed-off-by: Amit Barve <ambarve@microsoft.com>

cmd/containerd-shim-runhcs-v1/delete.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -17,8 +18,10 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oc"
+	cimlayer "github.com/Microsoft/hcsshim/internal/wclayer/cim"
 	"github.com/Microsoft/hcsshim/internal/winapi"
 )
 
@@ -123,6 +126,29 @@ The delete command will be executed in the container's bundle as its cwd.
 			fmt.Fprintf(os.Stderr, "failed to delete user %q: %v", username, err)
 		}
 
+		// cleanup the layers mounted for the container. We currently only handle cleanup of CimFS
+		// layers here.  First n-1 values should be the image layerFolders (topmost layer being at
+		// index 0) and the last entry should be the scratch layer
+		var layerFolders []string
+		f, err := os.Open(filepath.Join(bundleFlag, layersFile))
+		if err != nil {
+			if !errors.Is(err, os.ErrNotExist) {
+				fmt.Fprintf(os.Stderr, "open layers file: %s", err)
+			}
+		} else {
+			defer f.Close()
+			if err = json.NewDecoder(f).Decode(&layerFolders); err != nil {
+				fmt.Fprintf(os.Stderr, "decode layers json: %s", err)
+			}
+		}
+		if err == nil && cimlayer.IsCimLayer(layerFolders[0]) {
+			scratchLayerFolderPath := layerFolders[len(layerFolders)-1]
+			err = layers.ReleaseCimFSHostLayers(ctx, scratchLayerFolderPath, idFlag)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "cleanup container %q mounts: %s", idFlag, err)
+			}
+		}
+
 		if data, err := proto.Marshal(&task.DeleteResponse{
 			ExitedAt:   timestamppb.New(time.Now()),
 			ExitStatus: 255,

cmd/containerd-shim-runhcs-v1/main.go

@@ -29,6 +29,7 @@ import (
 
 const usage = ``
 const ttrpcAddressEnv = "TTRPC_ADDRESS"
+const layersFile = "layers.json"
 
 // Add a manifest to get proper Windows version detection.
 //go:generate go run github.com/josephspurrier/goversioninfo/cmd/goversioninfo -platform-specific

cmd/containerd-shim-runhcs-v1/service_internal.go

@@ -27,6 +27,13 @@ import (
 
 var empty = &emptypb.Empty{}
 
+// TODO(ambarve): Once we can vendor containerd 2.0 in hcsshim, we should directly reference these types from
+// containerd module
+const (
+	LegacyMountType string = "windows-layer"
+	CimFSMountType  string = "CimFS"
+)
+
 // getPod returns the pod this shim is tracking or else returns `nil`. It is the
 // callers responsibility to verify that `s.isSandbox == true` before calling
 // this method.
@@ -131,8 +138,11 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		// much of the creation logic in the shim. If we were given a
 		// rootfs mount, convert it to LayerFolders here.
 		m := req.Rootfs[0]
-		if m.Type != "windows-layer" {
+		if m.Type != LegacyMountType && m.Type != CimFSMountType {
 			return nil, fmt.Errorf("unsupported Windows mount type: %s", m.Type)
+		} else if m.Type == CimFSMountType && (shimOpts.SandboxIsolation == runhcsopts.Options_HYPERVISOR) {
+			// For CIMFS layers only process isolation is supported right now.
+			return nil, fmt.Errorf("cimfs doesn't support hyperv isolation")
 		}
 
 		source, parentLayerPaths, err := parseLegacyRootfsMount(m)
@@ -144,6 +154,20 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		spec.Windows.LayerFolders = append(spec.Windows.LayerFolders, parentLayerPaths...)
 		// Append the scratch
 		spec.Windows.LayerFolders = append(spec.Windows.LayerFolders, source)
+
+		if m.Type == CimFSMountType {
+			// write the layers to a file so that it can be used for proper cleanup during shim
+			// delete. We can't write to the config.json as it is read-only for shim.
+			f, err = os.Create(filepath.Join(req.Bundle, layersFile))
+			if err != nil {
+				return nil, err
+			}
+			if err := json.NewEncoder(f).Encode(spec.Windows.LayerFolders); err != nil {
+				f.Close()
+				return nil, err
+			}
+			f.Close()
+		}
 	}
 
 	// This is a Windows Argon make sure that we have a Root filled in.

computestorage/attach.go

@@ -38,3 +38,31 @@ func AttachLayerStorageFilter(ctx context.Context, layerPath string, layerData L
 	}
 	return nil
 }
+
+// AttachOverlayFilter sets up a filter of the given type on a writable container layer.  Currently the only
+// supported filter types are WCIFS & UnionFS (defined in internal/hcs/schema2/layer.go)
+//
+// `volumePath` is volume path at which writable layer is mounted. If the
+// path does not end in a `\` the platform will append it automatically.
+//
+// `layerData` is the parent read-only layer data.
+func AttachOverlayFilter(ctx context.Context, volumePath string, layerData LayerData) (err error) {
+	title := "hcsshim::AttachOverlayFilter"
+	ctx, span := oc.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("volumePath", volumePath),
+	)
+
+	bytes, err := json.Marshal(layerData)
+	if err != nil {
+		return err
+	}
+
+	err = hcsAttachOverlayFilter(volumePath, string(bytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to attach overlay filter")
+	}
+	return nil
+}

computestorage/detach.go

@@ -4,7 +4,9 @@ package computestorage
 
 import (
 	"context"
+	"encoding/json"
 
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/pkg/errors"
 	"go.opencensus.io/trace"
@@ -26,3 +28,27 @@ func DetachLayerStorageFilter(ctx context.Context, layerPath string) (err error)
 	}
 	return nil
 }
+
+// DetachOverlayFilter detaches the filter on a writable container layer.
+//
+// `volumePath` is a path to writable container volume.
+func DetachOverlayFilter(ctx context.Context, volumePath string, filterType hcsschema.FileSystemFilterType) (err error) {
+	title := "hcsshim::DetachOverlayFilter"
+	ctx, span := oc.StartSpan(ctx, title) //nolint:ineffassign,staticcheck
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(trace.StringAttribute("volumePath", volumePath))
+
+	layerData := LayerData{}
+	layerData.FilterType = filterType
+	bytes, err := json.Marshal(layerData)
+	if err != nil {
+		return err
+	}
+
+	err = hcsDetachOverlayFilter(volumePath, string(bytes))
+	if err != nil {
+		return errors.Wrap(err, "failed to detach overlay filter")
+	}
+	return nil
+}

computestorage/storage.go

@@ -19,14 +19,17 @@ import (
 //sys hcsFormatWritableLayerVhd(handle windows.Handle) (hr error) = computestorage.HcsFormatWritableLayerVhd?
 //sys hcsGetLayerVhdMountPath(vhdHandle windows.Handle, mountPath **uint16) (hr error) = computestorage.HcsGetLayerVhdMountPath?
 //sys hcsSetupBaseOSVolume(layerPath string, volumePath string, options string) (hr error) = computestorage.HcsSetupBaseOSVolume?
+//sys hcsAttachOverlayFilter(volumePath string, layerData string) (hr error) = computestorage.HcsAttachOverlayFilter?
+//sys hcsDetachOverlayFilter(volumePath string, layerData string) (hr error) = computestorage.HcsDetachOverlayFilter?
 
 type Version = hcsschema.Version
 type Layer = hcsschema.Layer
 
 // LayerData is the data used to describe parent layer information.
 type LayerData struct {
-	SchemaVersion Version `json:"SchemaVersion,omitempty"`
-	Layers        []Layer `json:"Layers,omitempty"`
+	SchemaVersion Version                        `json:"SchemaVersion,omitempty"`
+	Layers        []Layer                        `json:"Layers,omitempty"`
+	FilterType    hcsschema.FileSystemFilterType `json:"FilterType,omitempty"`
 }
 
 // ExportLayerOptions are the set of options that are used with the `computestorage.HcsExportLayer` syscall.

internal/hcs/schema2/layer.go

@@ -9,6 +9,13 @@
 
 package hcsschema
 
+type FileSystemFilterType string
+
+const (
+	UnionFS FileSystemFilterType = "UnionFS"
+	WCIFS   FileSystemFilterType = "WCIFS"
+)
+
 type Layer struct {
 	Id string `json:"Id,omitempty"`
 

internal/hcsoci/hcsdoc_wcow.go

@@ -26,7 +26,6 @@ import (
 	"github.com/Microsoft/hcsshim/internal/processorinfo"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 	"github.com/Microsoft/hcsshim/internal/uvmfolder"
-	"github.com/Microsoft/hcsshim/internal/wclayer"
 	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
 )
@@ -354,13 +353,13 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter
 	}
 
 	if coi.isV2Argon() || coi.isV1Argon() { // Argon v1 or v2
-		for _, layerPath := range coi.Spec.Windows.LayerFolders[:len(coi.Spec.Windows.LayerFolders)-1] {
-			layerID, err := wclayer.LayerID(ctx, layerPath)
-			if err != nil {
-				return nil, nil, err
-			}
-			v1.Layers = append(v1.Layers, schema1.Layer{ID: layerID.String(), Path: layerPath})
-			v2Container.Storage.Layers = append(v2Container.Storage.Layers, hcsschema.Layer{Id: layerID.String(), Path: layerPath})
+		mountedLayers, err := layers.ToHostHcsSchemaLayers(ctx, coi.ID, coi.Spec.Windows.LayerFolders[:len(coi.Spec.Windows.LayerFolders)-1])
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, ml := range mountedLayers {
+			v1.Layers = append(v1.Layers, schema1.Layer{ID: ml.Id, Path: ml.Path})
+			v2Container.Storage.Layers = append(v2Container.Storage.Layers, ml)
 		}
 	}