Add abstractions for direct HCS interactions

internal/uvm/virtual_device.go

@@ -95,7 +95,8 @@ func (uvm *UtilityVM) AssignDevice(ctx context.Context, deviceID string) (*VPCID
 	request := &hcsschema.ModifySettingRequest{
 		ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmBusGUID),
 		RequestType:  requesttype.Add,
-		Settings:     targetDevice}
+		Settings:     targetDevice,
+	}
 
 	// WCOW (when supported) does not require a guest request as part of the
 	// device assignment

internal/vm/builder.go

@@ -0,0 +1,74 @@
+package vm
+
+import (
+	"context"
+)
+
+type UVMBuilder interface {
+	// Create will create the Utility VM in a paused/powered off state with whatever is present in the implementation
+	// of the interfaces config at the time of the call.
+	Create(ctx context.Context) (UVM, error)
+}
+
+type MemoryBackingType uint8
+
+const (
+	MemoryBackingTypeVirtual MemoryBackingType = iota
+	MemoryBackingTypePhysical
+)
+
+// MemoryConfig holds the memory options that should be configurable for a Utility VM.
+type MemoryConfig struct {
+	BackingType     MemoryBackingType
+	DeferredCommit  bool
+	HotHint         bool
+	ColdHint        bool
+	ColdDiscardHint bool
+}
+
+// MemoryManager handles setting and managing memory configurations for the Utility VM.
+type MemoryManager interface {
+	// SetMemoryLimit sets the amount of memory in megabytes that the Utility VM will be assigned.
+	SetMemoryLimit(memoryMB uint64) error
+	// SetMemoryConfig sets an array of different memory configuration options available. This includes things like the
+	// type of memory to back the VM (virtual/physical).
+	SetMemoryConfig(config *MemoryConfig) error
+	// SetMMIOConfig sets memory mapped IO configurations for the Utility VM.
+	SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error
+}
+
+// ProcessorManager handles setting and managing processor configurations for the Utility VM.
+type ProcessorManager interface {
+	// SetProcessorCount sets the number of virtual processors that will be assigned to the Utility VM.
+	SetProcessorCount(count uint32) error
+}
+
+// SerialManager manages setting up serial consoles for the Utility VM.
+type SerialManager interface {
+	// SetSerialConsole sets up a serial console for `port`. Output will be relayed to the listener specified
+	// by `listenerPath`. For HCS `listenerPath` this is expected to be a path to a named pipe.
+	SetSerialConsole(port uint32, listenerPath string) error
+}
+
+// BootManager manages boot configurations for the Utility VM.
+type BootManager interface {
+	// SetUEFIBoot sets UEFI configurations for booting a Utility VM.
+	SetUEFIBoot(dir string, path string, args string) error
+	// SetLinuxKernelDirectBoot sets Linux direct boot configurations for booting a Utility VM.
+	SetLinuxKernelDirectBoot(kernel string, initRD string, cmd string) error
+}
+
+// StorageQosManager manages setting storage limits on the Utility VM.
+type StorageQosManager interface {
+	// SetStorageQos sets storage related options for the Utility VM
+	SetStorageQos(iopsMaximum int64, bandwidthMaximum int64) error
+}
+
+// WindowsConfigManager manages options specific to a Windows host (cpugroups etc.)
+type WindowsConfigManager interface {
+	// SetCPUGroup sets the CPU group that the Utility VM will belong to on a Windows host.
+	SetCPUGroup(id string) error
+}
+
+// LinuxConfigManager manages options specific to a Linux host.
+type LinuxConfigManager interface{}

internal/vm/hcs/boot.go

@@ -0,0 +1,31 @@
+package hcs
+
+import (
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/Microsoft/hcsshim/osversion"
+	"github.com/pkg/errors"
+)
+
+func (uvmb *utilityVMBuilder) SetUEFIBoot(dir string, path string, args string) error {
+	uvmb.doc.VirtualMachine.Chipset.Uefi = &hcsschema.Uefi{
+		BootThis: &hcsschema.UefiBootEntry{
+			DevicePath:    path,
+			DeviceType:    "VmbFs",
+			VmbFsRootPath: dir,
+			OptionalData:  args,
+		},
+	}
+	return nil
+}
+
+func (uvmb *utilityVMBuilder) SetLinuxKernelDirectBoot(kernel string, initRD string, cmd string) error {
+	if osversion.Get().Build < 18286 {
+		return errors.New("Linux kernel direct boot requires at least Windows version 18286")
+	}
+	uvmb.doc.VirtualMachine.Chipset.LinuxKernelDirect = &hcsschema.LinuxKernelDirect{
+		KernelFilePath: kernel,
+		InitRdPath:     initRD,
+		KernelCmdLine:  cmd,
+	}
+	return nil
+}

internal/vm/hcs/builder.go

@@ -0,0 +1,93 @@
+package hcs
+
+import (
+	"context"
+
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/Microsoft/hcsshim/internal/schemaversion"
+	"github.com/Microsoft/hcsshim/internal/vm"
+	"github.com/pkg/errors"
+)
+
+type utilityVMBuilder struct {
+	id      string
+	guestOS vm.GuestOS
+	doc     *hcsschema.ComputeSystem
+}
+
+func NewUVMBuilder(id string, owner string, guestOS vm.GuestOS) (vm.UVMBuilder, error) {
+	doc := &hcsschema.ComputeSystem{
+		Owner:                             owner,
+		SchemaVersion:                     schemaversion.SchemaV21(),
+		ShouldTerminateOnLastHandleClosed: true,
+		VirtualMachine: &hcsschema.VirtualMachine{
+			StopOnReset: true,
+			Chipset:     &hcsschema.Chipset{},
+			ComputeTopology: &hcsschema.Topology{
+				Memory: &hcsschema.Memory2{
+					AllowOvercommit: true,
+				},
+				Processor: &hcsschema.Processor2{},
+			},
+			Devices: &hcsschema.Devices{
+				HvSocket: &hcsschema.HvSocket2{
+					HvSocketConfig: &hcsschema.HvSocketSystemConfig{
+						// Allow administrators and SYSTEM to bind to vsock sockets
+						// so that we can create a GCS log socket.
+						DefaultBindSecurityDescriptor: "D:P(A;;FA;;;SY)(A;;FA;;;BA)",
+					},
+				},
+			},
+		},
+	}
+
+	switch guestOS {
+	case vm.Windows:
+		doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{}
+	case vm.Linux:
+		doc.VirtualMachine.Devices.Plan9 = &hcsschema.Plan9{}
+	default:
+		return nil, vm.ErrUnknownGuestOS
+	}
+
+	return &utilityVMBuilder{
+		id:      id,
+		guestOS: guestOS,
+		doc:     doc,
+	}, nil
+}
+
+func (uvmb *utilityVMBuilder) Create(ctx context.Context) (_ vm.UVM, err error) {
+	cs, err := hcs.CreateComputeSystem(ctx, uvmb.id, uvmb.doc)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create hcs compute system")
+	}
+
+	defer func() {
+		if err != nil {
+			_ = cs.Terminate(ctx)
+			_ = cs.Wait()
+		}
+	}()
+
+	backingType := vm.MemoryBackingTypeVirtual
+	if !uvmb.doc.VirtualMachine.ComputeTopology.Memory.AllowOvercommit {
+		backingType = vm.MemoryBackingTypePhysical
+	}
+
+	uvm := &utilityVM{
+		id:          uvmb.id,
+		guestOS:     uvmb.guestOS,
+		cs:          cs,
+		backingType: backingType,
+		state:       vm.StateCreated,
+	}
+
+	properties, err := cs.Properties(ctx)
+	if err != nil {
+		return nil, err
+	}
+	uvm.vmID = properties.RuntimeID
+	return uvm, nil
+}

internal/vm/hcs/hcs.go

@@ -0,0 +1,75 @@
+package hcs
+
+import (
+	"context"
+	"sync"
+
+	"github.com/Microsoft/go-winio/pkg/guid"
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/Microsoft/hcsshim/internal/vm"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
+)
+
+type utilityVM struct {
+	id           string
+	state        vm.State
+	guestOS      vm.GuestOS
+	cs           *hcs.System
+	backingType  vm.MemoryBackingType
+	vmmemProcess windows.Handle
+	vmmemErr     error
+	vmmemOnce    sync.Once
+	vmID         guid.GUID
+}
+
+func (uvm *utilityVM) ID() string {
+	return uvm.id
+}
+
+func (uvm *utilityVM) Start(ctx context.Context) (err error) {
+	if err := uvm.cs.Start(ctx); err != nil {
+		return errors.Wrap(err, "failed to start utility VM")
+	}
+	return nil
+}
+
+func (uvm *utilityVM) Stop(ctx context.Context) error {
+	if err := uvm.cs.Terminate(ctx); err != nil {
+		return errors.Wrap(err, "failed to terminate utility VM")
+	}
+	return nil
+}
+
+func (uvm *utilityVM) Pause(ctx context.Context) error {
+	if err := uvm.cs.Pause(ctx); err != nil {
+		return errors.Wrap(err, "failed to pause utility VM")
+	}
+	return nil
+}
+
+func (uvm *utilityVM) Resume(ctx context.Context) error {
+	if err := uvm.cs.Resume(ctx); err != nil {
+		return errors.Wrap(err, "failed to resume utility VM")
+	}
+	return nil
+}
+
+func (uvm *utilityVM) Save(ctx context.Context) error {
+	saveOptions := hcsschema.SaveOptions{
+		SaveType: "AsTemplate",
+	}
+	if err := uvm.cs.Save(ctx, saveOptions); err != nil {
+		return errors.Wrap(err, "failed to save utility VM state")
+	}
+	return nil
+}
+
+func (uvm *utilityVM) Wait() error {
+	return uvm.cs.Wait()
+}
+
+func (uvm *utilityVM) ExitError() error {
+	return uvm.cs.ExitError()
+}

internal/vm/hcs/memory.go

@@ -0,0 +1,28 @@
+package hcs
+
+import (
+	"github.com/Microsoft/hcsshim/internal/vm"
+)
+
+func (uvmb *utilityVMBuilder) SetMemoryLimit(memoryMB uint64) error {
+	uvmb.doc.VirtualMachine.ComputeTopology.Memory.SizeInMB = memoryMB
+	return nil
+}
+
+func (uvmb *utilityVMBuilder) SetMemoryConfig(config *vm.MemoryConfig) error {
+	memory := uvmb.doc.VirtualMachine.ComputeTopology.Memory
+	memory.AllowOvercommit = config.BackingType == vm.MemoryBackingTypeVirtual
+	memory.EnableDeferredCommit = config.DeferredCommit
+	memory.EnableHotHint = config.HotHint
+	memory.EnableColdHint = config.ColdHint
+	memory.EnableColdDiscardHint = config.ColdDiscardHint
+	return nil
+}
+
+func (uvmb *utilityVMBuilder) SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error {
+	memory := uvmb.doc.VirtualMachine.ComputeTopology.Memory
+	memory.LowMMIOGapInMB = lowGapMB
+	memory.HighMMIOBaseInMB = highBaseMB
+	memory.HighMMIOGapInMB = highGapMB
+	return nil
+}

internal/vm/hcs/network.go

@@ -0,0 +1,34 @@
+package hcs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/internal/hcs/resourcepaths"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/Microsoft/hcsshim/internal/requesttype"
+)
+
+func (uvm *utilityVM) AddNIC(ctx context.Context, nicID, endpointID, macAddr string) error {
+	request := hcsschema.ModifySettingRequest{
+		RequestType:  requesttype.Add,
+		ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, nicID),
+		Settings: hcsschema.NetworkAdapter{
+			EndpointId: endpointID,
+			MacAddress: macAddr,
+		},
+	}
+	return uvm.cs.Modify(ctx, request)
+}
+
+func (uvm *utilityVM) RemoveNIC(ctx context.Context, nicID, endpointID, macAddr string) error {
+	request := hcsschema.ModifySettingRequest{
+		RequestType:  requesttype.Remove,
+		ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, nicID),
+		Settings: hcsschema.NetworkAdapter{
+			EndpointId: endpointID,
+			MacAddress: macAddr,
+		},
+	}
+	return uvm.cs.Modify(ctx, request)
+}

internal/vm/hcs/pci.go

@@ -0,0 +1,33 @@
+package hcs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/internal/hcs/resourcepaths"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/Microsoft/hcsshim/internal/requesttype"
+)
+
+func (uvm *utilityVM) AddDevice(ctx context.Context, instanceID, vmbusGUID string) error {
+	request := &hcsschema.ModifySettingRequest{
+		ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmbusGUID),
+		RequestType:  requesttype.Add,
+		Settings: hcsschema.VirtualPciDevice{
+			Functions: []hcsschema.VirtualPciFunction{
+				{
+					DeviceInstancePath: instanceID,
+				},
+			},
+		},
+	}
+	return uvm.cs.Modify(ctx, request)
+}
+
+func (uvm *utilityVM) RemoveDevice(ctx context.Context, instanceID, vmbusGUID string) error {
+	request := &hcsschema.ModifySettingRequest{
+		ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmbusGUID),
+		RequestType:  requesttype.Remove,
+	}
+	return uvm.cs.Modify(ctx, request)
+}