Add support for assigning cpu group on creation

In recent builds of Windows there was support added to the HCS to allow assigning
a cpugroup at creation time of the VM instead of afterwards. The current approach in this
repo of adding a vm after start was only a workaround as this wasn't supported at the time.
The current approach isn;t ideal due to some wonky behavior on machines with
multiple NUMA nodes as we can suffer performance penalties because of remote memory access on
machines with > 1 node when adding a VM after start.

Signed-off-by: Daniel Canter <dcanter@microsoft.com>

Author: dcantah

internal/cpugroup/cpugroup.go

@@ -3,12 +3,12 @@ package cpugroup
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/Microsoft/hcsshim/internal/hcs"
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
+	"github.com/pkg/errors"
 )
 
 const NullGroupID = "00000000-0000-0000-0000-000000000000"
@@ -48,7 +48,7 @@ func Create(ctx context.Context, id string, logicalProcessors []uint32) error {
 		LogicalProcessorCount: uint32(len(logicalProcessors)),
 	}
 	if err := modifyCPUGroupRequest(ctx, operation, details); err != nil {
-		return fmt.Errorf("failed to make cpugroups CreateGroup request for details %+v with: %s", details, err)
+		return errors.Wrapf(err, "failed to make cpugroups CreateGroup request for details %+v", details)
 	}
 	return nil
 }
@@ -65,7 +65,7 @@ func getCPUGroupConfig(ctx context.Context, id string) (*hcsschema.CpuGroupConfi
 	}
 	groupConfigs := &hcsschema.CpuGroupConfigurations{}
 	if err := json.Unmarshal(cpuGroupsPresent.Properties[0], groupConfigs); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal host cpugroups: %v", err)
+		return nil, errors.Wrap(err, "failed to unmarshal host cpugroups")
 	}
 
 	for _, c := range groupConfigs.CpuGroups {

internal/cpugroup/cpugroup_test.go

@@ -14,14 +14,17 @@ func TestCPUGroupCreateWithIDAndDelete(t *testing.T) {
 	lps := []uint32{0, 1}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+
 	id, err := guid.NewV4()
 	if err != nil {
 		t.Fatalf("failed to create cpugroup guid with: %v", err)
 	}
+
 	err = Create(ctx, id.String(), lps)
 	if err != nil {
 		t.Fatalf("failed to create cpugroup %s with: %v", id.String(), err)
 	}
+
 	defer func() {
 		if err := Delete(ctx, id.String()); err != nil {
 			t.Fatalf("failed to delete cpugroup %s with: %v", id.String(), err)

internal/hcs/schema2/processor_2.go

@@ -3,7 +3,7 @@
  *
  * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
  *
- * API version: 2.1
+ * API version: 2.5
  * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
  */
 
@@ -17,4 +17,7 @@ type Processor2 struct {
 	Weight int32 `json:"Weight,omitempty"`
 
 	ExposeVirtualizationExtensions bool `json:"ExposeVirtualizationExtensions,omitempty"`
+
+	// An optional object that configures the CPU Group to which a Virtual Machine is going to bind to.
+	CpuGroup *CpuGroup `json:"CpuGroup,omitempty"`
 }

internal/uvm/cpugroups.go renamed to internal/uvm/cpugroup.go

@@ -10,6 +10,11 @@ import (
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 )
 
+// Build that assigning a cpu group on creation of a vm is supported
+const cpuGroupCreateBuild = 20124
+
+var cpuGroupCreateNotSupported = fmt.Errorf("cpu group assignment requires a build of %d or higher", cpuGroupCreateBuild)
+
 // ReleaseCPUGroup unsets the cpugroup from the VM
 func (uvm *UtilityVM) ReleaseCPUGroup(ctx context.Context) error {
 	groupID := uvm.cpuGroupID

internal/uvm/create.go

@@ -244,9 +244,6 @@ func (uvm *UtilityVM) Close() (err error) {
 	windows.Close(uvm.vmmemProcess)
 
 	if uvm.hcsSystem != nil {
-		if err := uvm.ReleaseCPUGroup(ctx); err != nil {
-			log.G(ctx).WithError(err).Warn("failed to release VM resource")
-		}
 		_ = uvm.hcsSystem.Terminate(ctx)
 		_ = uvm.Wait()
 	}

internal/uvm/create_lcow.go

@@ -204,6 +204,19 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
 	// Align the requested memory size.
 	memorySizeInMB := uvm.normalizeMemorySize(ctx, opts.MemorySizeInMB)
 
+	processor := &hcsschema.Processor2{
+		Count:  uvm.processorCount,
+		Limit:  opts.ProcessorLimit,
+		Weight: opts.ProcessorWeight,
+	}
+	// We can set a cpu group for the VM at creation time in recent builds.
+	if opts.CPUGroupID != "" {
+		if osversion.Build() < cpuGroupCreateBuild {
+			return nil, cpuGroupCreateNotSupported
+		}
+		processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID}
+	}
+
 	doc := &hcsschema.ComputeSystem{
 		Owner:                             uvm.owner,
 		SchemaVersion:                     schemaversion.SchemaV21(),
@@ -221,11 +234,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
 					HighMMIOBaseInMB:      opts.HighMMIOBaseInMB,
 					HighMMIOGapInMB:       opts.HighMMIOGapInMB,
 				},
-				Processor: &hcsschema.Processor2{
-					Count:  uvm.processorCount,
-					Limit:  opts.ProcessorLimit,
-					Weight: opts.ProcessorWeight,
-				},
+				Processor: processor,
 			},
 			Devices: &hcsschema.Devices{
 				HvSocket: &hcsschema.HvSocket2{
@@ -381,6 +390,8 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
 		return nil, fmt.Errorf("error while creating the compute system: %s", err)
 	}
 
+	uvm.cpuGroupID = opts.CPUGroupID
+
 	// Cerate a socket to inject entropy during boot.
 	uvm.entropyListener, err = uvm.listenVsock(entropyVsockPort)
 	if err != nil {

internal/uvm/create_wcow.go

@@ -19,6 +19,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/uvmfolder"
 	"github.com/Microsoft/hcsshim/internal/wclayer"
 	"github.com/Microsoft/hcsshim/internal/wcow"
+	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/containerd/ttrpc"
 	"github.com/pkg/errors"
 	"go.opencensus.io/trace"
@@ -122,6 +123,19 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv
 		}
 	}
 
+	processor := &hcsschema.Processor2{
+		Count:  uvm.processorCount,
+		Limit:  opts.ProcessorLimit,
+		Weight: opts.ProcessorWeight,
+	}
+	// We can set a cpu group for the VM at creation time in recent builds.
+	if opts.CPUGroupID != "" {
+		if osversion.Build() < cpuGroupCreateBuild {
+			return nil, cpuGroupCreateNotSupported
+		}
+		processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID}
+	}
+
 	doc := &hcsschema.ComputeSystem{
 		Owner:                             uvm.owner,
 		SchemaVersion:                     schemaversion.SchemaV21(),
@@ -148,11 +162,7 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv
 					HighMMIOBaseInMB:     opts.HighMMIOBaseInMB,
 					HighMMIOGapInMB:      opts.HighMMIOGapInMB,
 				},
-				Processor: &hcsschema.Processor2{
-					Count:  uvm.processorCount,
-					Limit:  opts.ProcessorLimit,
-					Weight: opts.ProcessorWeight,
-				},
+				Processor: processor,
 			},
 			Devices: &hcsschema.Devices{
 				HvSocket: &hcsschema.HvSocket2{
@@ -319,6 +329,8 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error
 		return nil, fmt.Errorf("error while creating the compute system: %s", err)
 	}
 
+	uvm.cpuGroupID = opts.CPUGroupID
+
 	// All clones MUST use external gcs connection
 	if opts.ExternalGuestConnection {
 		if err = uvm.startExternalGcsListener(ctx); err != nil {

internal/uvm/start.go

@@ -205,13 +205,6 @@ func (uvm *UtilityVM) Start(ctx context.Context) (err error) {
 		}
 	}()
 
-	// assign the VM to the cpugroup specified, if any
-	if uvm.cpuGroupID != "" {
-		if err := uvm.SetCPUGroup(ctx, uvm.cpuGroupID); err != nil {
-			return err
-		}
-	}
-
 	// Start waiting on the utility VM.
 	uvm.exitCh = make(chan struct{})
 	go func() {