pod: hot add/remove vCPUs before starting containers

depending of the container's state, pod's resources are updated.
for example, if the container has a CPU constraint of 4 vCPU and it is stopped,
then these 4 vCPUs are removed.
another example, if the container is ready to be started and it has a CPU
constraint of 3 vCPUs, 3 vCPUs are added and the constraint is sent to the
agent to limit CPU usage of the container.
With this patch the fractional part of the CPU constraint is also honoured,
for example, if the container has a CPU constraint of 5.5, then 6 vCPUs are
added to the POD and a constraint of 5.5 CPUs is applied to the container

Signed-off-by: Julio Montes <julio.montes@intel.com>

api_test.go

@@ -883,6 +883,7 @@ func TestStatusPodSuccessfulStateReady(t *testing.T) {
 		DefaultMemSz:      defaultMemSzMiB,
 		DefaultBridges:    defaultBridges,
 		BlockDeviceDriver: defaultBlockDriver,
+		DefaultMaxVCPUs:   defaultMaxQemuVCPUs,
 	}
 
 	expectedStatus := PodStatus{
@@ -938,6 +939,7 @@ func TestStatusPodSuccessfulStateRunning(t *testing.T) {
 		DefaultMemSz:      defaultMemSzMiB,
 		DefaultBridges:    defaultBridges,
 		BlockDeviceDriver: defaultBlockDriver,
+		DefaultMaxVCPUs:   defaultMaxQemuVCPUs,
 	}
 
 	expectedStatus := PodStatus{

container.go

@@ -58,6 +58,16 @@ type ContainerStatus struct {
 	Annotations map[string]string
 }
 
+// ContainerResources describes container resources
+type ContainerResources struct {
+	// CPUQuota specifies the total amount of time in microseconds
+	// The number of microseconds per CPUPeriod that the container is guaranteed CPU access
+	CPUQuota int64
+
+	// CPUPeriod specifies the CPU CFS scheduler period of time in microseconds
+	CPUPeriod uint64
+}
+
 // ContainerConfig describes one container runtime configuration.
 type ContainerConfig struct {
 	ID string
@@ -80,6 +90,9 @@ type ContainerConfig struct {
 
 	// Device configuration for devices that must be available within the container.
 	DeviceInfos []DeviceInfo
+
+	// Resources container resources
+	Resources ContainerResources
 }
 
 // valid checks that the container configuration is valid.
@@ -436,6 +449,10 @@ func createContainer(pod *Pod, contConfig ContainerConfig) (*Container, error) {
 		return nil, err
 	}
 
+	if err := c.addResources(); err != nil {
+		return nil, err
+	}
+
 	// Deduce additional system mount info that should be handled by the agent
 	// inside the VM
 	c.getSystemMountInfo()
@@ -591,6 +608,10 @@ func (c *Container) stop() error {
 		return err
 	}
 
+	if err := c.removeResources(); err != nil {
+		return err
+	}
+
 	if err := c.detachDevices(); err != nil {
 		return err
 	}
@@ -754,3 +775,37 @@ func (c *Container) detachDevices() error {
 
 	return nil
 }
+
+func (c *Container) addResources() error {
+	//TODO add support for memory, Issue: https://github.com/containers/virtcontainers/issues/578
+	if c.config == nil {
+		return nil
+	}
+
+	vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod)
+	if vCPUs != 0 {
+		virtLog.Debugf("hot adding %d vCPUs", vCPUs)
+		if err := c.pod.hypervisor.hotplugAddDevice(uint32(vCPUs), cpuDev); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Container) removeResources() error {
+	//TODO add support for memory, Issue: https://github.com/containers/virtcontainers/issues/578
+	if c.config == nil {
+		return nil
+	}
+
+	vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod)
+	if vCPUs != 0 {
+		virtLog.Debugf("hot removing %d vCPUs", vCPUs)
+		if err := c.pod.hypervisor.hotplugRemoveDevice(uint32(vCPUs), cpuDev); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

container_test.go

@@ -26,6 +26,7 @@ import (
 	"syscall"
 	"testing"
 
+	vcAnnotations "github.com/containers/virtcontainers/pkg/annotations"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -281,3 +282,53 @@ func TestCheckPodRunningSuccessful(t *testing.T) {
 	err := c.checkPodRunning("test_cmd")
 	assert.Nil(t, err, "%v", err)
 }
+
+func TestContainerAddResources(t *testing.T) {
+	assert := assert.New(t)
+
+	c := &Container{}
+	err := c.addResources()
+	assert.Nil(err)
+
+	c.config = &ContainerConfig{Annotations: make(map[string]string)}
+	c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodSandbox)
+	err = c.addResources()
+	assert.Nil(err)
+
+	c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodContainer)
+	err = c.addResources()
+	assert.Nil(err)
+
+	c.config.Resources = ContainerResources{
+		CPUQuota:  5000,
+		CPUPeriod: 1000,
+	}
+	c.pod = &Pod{hypervisor: &mockHypervisor{}}
+	err = c.addResources()
+	assert.Nil(err)
+}
+
+func TestContainerRemoveResources(t *testing.T) {
+	assert := assert.New(t)
+
+	c := &Container{}
+	err := c.addResources()
+	assert.Nil(err)
+
+	c.config = &ContainerConfig{Annotations: make(map[string]string)}
+	c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodSandbox)
+	err = c.removeResources()
+	assert.Nil(err)
+
+	c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodContainer)
+	err = c.removeResources()
+	assert.Nil(err)
+
+	c.config.Resources = ContainerResources{
+		CPUQuota:  5000,
+		CPUPeriod: 1000,
+	}
+	c.pod = &Pod{hypervisor: &mockHypervisor{}}
+	err = c.removeResources()
+	assert.Nil(err)
+}

hyperstart_agent.go

@@ -424,6 +424,13 @@ func (h *hyper) startOneContainer(pod Pod, c *Container) error {
 		Process: process,
 	}
 
+	if c.config.Resources.CPUQuota != 0 && c.config.Resources.CPUPeriod != 0 {
+		container.Constraints = hyperstart.Constraints{
+			CPUQuota:  c.config.Resources.CPUQuota,
+			CPUPeriod: c.config.Resources.CPUPeriod,
+		}
+	}
+
 	container.SystemMountsInfo.BindMountDev = c.systemMountsInfo.BindMountDev
 
 	if c.state.Fstype != "" {

hypervisor_test.go

@@ -181,6 +181,7 @@ func TestHypervisorConfigDefaults(t *testing.T) {
 		DefaultMemSz:      defaultMemSzMiB,
 		DefaultBridges:    defaultBridges,
 		BlockDeviceDriver: defaultBlockDriver,
+		DefaultMaxVCPUs:   defaultMaxQemuVCPUs,
 	}
 	if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false {
 		t.Fatal()

pkg/hyperstart/types.go

@@ -202,6 +202,16 @@ type SystemMountsInfo struct {
 	DevShmSize int `json:"devShmSize"`
 }
 
+// Constraints describes the constrains for a container
+type Constraints struct {
+	// CPUQuota specifies the total amount of time in microseconds
+	// The number of microseconds per CPUPeriod that the container is guaranteed CPU access
+	CPUQuota int64
+
+	// CPUPeriod specifies the CPU CFS scheduler period of time in microseconds
+	CPUPeriod uint64
+}
+
 // Container describes a container running on a pod.
 type Container struct {
 	ID               string              `json:"id"`
@@ -216,6 +226,7 @@ type Container struct {
 	RestartPolicy    string              `json:"restartPolicy"`
 	Initialize       bool                `json:"initialize"`
 	SystemMountsInfo SystemMountsInfo    `json:"systemMountsInfo"`
+	Constraints      Constraints         `json:"constraints"`
 }
 
 // IPAddress describes an IP address and its network mask.

pkg/oci/utils.go

@@ -454,13 +454,7 @@ func vmConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.Resources, error)
 			return vc.Resources{}, fmt.Errorf("Invalid OCI cpu period %d", period)
 		}
 
-		// Use some math magic to round up to the nearest whole vCPU
-		// (that is, a partial part of a quota request ends up assigning
-		// a whole vCPU, for instance, a request of 1.5 'cpu quotas'
-		// will give 2 vCPUs).
-		// This also has the side effect that we will always allocate
-		// at least 1 vCPU.
-		resources.VCPUs = uint((uint64(quota) + (period - 1)) / period)
+		resources.VCPUs = vc.ConstraintsToVCPUs(quota, period)
 	}
 
 	return resources, nil
@@ -587,6 +581,14 @@ func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, det
 		return vc.ContainerConfig{}, err
 	}
 
+	var resources vc.ContainerResources
+	if ocispec.Linux.Resources.CPU != nil &&
+		ocispec.Linux.Resources.CPU.Quota != nil &&
+		ocispec.Linux.Resources.CPU.Period != nil {
+		resources.CPUQuota = *ocispec.Linux.Resources.CPU.Quota
+		resources.CPUPeriod = *ocispec.Linux.Resources.CPU.Period
+	}
+
 	containerConfig := vc.ContainerConfig{
 		ID:             cid,
 		RootFs:         rootfs,
@@ -598,6 +600,7 @@ func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, det
 		},
 		Mounts:      containerMounts(ocispec),
 		DeviceInfos: deviceInfos,
+		Resources:   resources,
 	}
 
 	cType, err := ocispec.ContainerType()

qemu_test.go

@@ -37,6 +37,7 @@ func newQemuConfig() HypervisorConfig {
 		DefaultMemSz:      defaultMemSzMiB,
 		DefaultBridges:    defaultBridges,
 		BlockDeviceDriver: defaultBlockDriver,
+		DefaultMaxVCPUs:   defaultMaxQemuVCPUs,
 	}
 }
 

utils.go

@@ -96,3 +96,18 @@ func writeToFile(path string, data []byte) error {
 
 	return nil
 }
+
+// ConstraintsToVCPUs converts CPU quota and period to vCPUs
+func ConstraintsToVCPUs(quota int64, period uint64) uint {
+	if quota != 0 && period != 0 {
+		// Use some math magic to round up to the nearest whole vCPU
+		// (that is, a partial part of a quota request ends up assigning
+		// a whole vCPU, for instance, a request of 1.5 'cpu quotas'
+		// will give 2 vCPUs).
+		// This also has the side effect that we will always allocate
+		// at least 1 vCPU.
+		return uint((uint64(quota) + (period - 1)) / period)
+	}
+
+	return 0
+}

utils_test.go

@@ -162,3 +162,20 @@ func TestWriteToFile(t *testing.T) {
 
 	assert.True(t, reflect.DeepEqual(testData, data))
 }
+
+func TestConstraintsToVCPUs(t *testing.T) {
+	assert := assert.New(t)
+
+	vcpus := ConstraintsToVCPUs(0, 100)
+	assert.Zero(vcpus)
+
+	vcpus = ConstraintsToVCPUs(100, 0)
+	assert.Zero(vcpus)
+
+	expectedVCPUs := uint(4)
+	vcpus = ConstraintsToVCPUs(4000, 1000)
+	assert.Equal(expectedVCPUs, vcpus)
+
+	vcpus = ConstraintsToVCPUs(4000, 1200)
+	assert.Equal(expectedVCPUs, vcpus)
+}