diff --git a/api_test.go b/api_test.go index 92a20f91..14c952d5 100644 --- a/api_test.go +++ b/api_test.go @@ -883,6 +883,7 @@ func TestStatusPodSuccessfulStateReady(t *testing.T) { DefaultMemSz: defaultMemSzMiB, DefaultBridges: defaultBridges, BlockDeviceDriver: defaultBlockDriver, + DefaultMaxVCPUs: defaultMaxQemuVCPUs, } expectedStatus := PodStatus{ @@ -938,6 +939,7 @@ func TestStatusPodSuccessfulStateRunning(t *testing.T) { DefaultMemSz: defaultMemSzMiB, DefaultBridges: defaultBridges, BlockDeviceDriver: defaultBlockDriver, + DefaultMaxVCPUs: defaultMaxQemuVCPUs, } expectedStatus := PodStatus{ diff --git a/container.go b/container.go index a1d302e7..ebd277b5 100644 --- a/container.go +++ b/container.go @@ -58,6 +58,16 @@ type ContainerStatus struct { Annotations map[string]string } +// ContainerResources describes container resources +type ContainerResources struct { + // CPUQuota specifies the total amount of time in microseconds + // The number of microseconds per CPUPeriod that the container is guaranteed CPU access + CPUQuota int64 + + // CPUPeriod specifies the CPU CFS scheduler period of time in microseconds + CPUPeriod uint64 +} + // ContainerConfig describes one container runtime configuration. type ContainerConfig struct { ID string @@ -80,6 +90,9 @@ type ContainerConfig struct { // Device configuration for devices that must be available within the container. DeviceInfos []DeviceInfo + + // Resources container resources + Resources ContainerResources } // valid checks that the container configuration is valid. @@ -436,6 +449,10 @@ func createContainer(pod *Pod, contConfig ContainerConfig) (*Container, error) { return nil, err } + if err := c.addResources(); err != nil { + return nil, err + } + // Deduce additional system mount info that should be handled by the agent // inside the VM c.getSystemMountInfo() @@ -591,6 +608,10 @@ func (c *Container) stop() error { return err } + if err := c.removeResources(); err != nil { + return err + } + if err := c.detachDevices(); err != nil { return err } @@ -754,3 +775,37 @@ func (c *Container) detachDevices() error { return nil } + +func (c *Container) addResources() error { + //TODO add support for memory, Issue: https://github.com/containers/virtcontainers/issues/578 + if c.config == nil { + return nil + } + + vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod) + if vCPUs != 0 { + virtLog.Debugf("hot adding %d vCPUs", vCPUs) + if err := c.pod.hypervisor.hotplugAddDevice(uint32(vCPUs), cpuDev); err != nil { + return err + } + } + + return nil +} + +func (c *Container) removeResources() error { + //TODO add support for memory, Issue: https://github.com/containers/virtcontainers/issues/578 + if c.config == nil { + return nil + } + + vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod) + if vCPUs != 0 { + virtLog.Debugf("hot removing %d vCPUs", vCPUs) + if err := c.pod.hypervisor.hotplugRemoveDevice(uint32(vCPUs), cpuDev); err != nil { + return err + } + } + + return nil +} diff --git a/container_test.go b/container_test.go index 4aef3ec4..a2e7a063 100644 --- a/container_test.go +++ b/container_test.go @@ -26,6 +26,7 @@ import ( "syscall" "testing" + vcAnnotations "github.com/containers/virtcontainers/pkg/annotations" "github.com/stretchr/testify/assert" ) @@ -281,3 +282,53 @@ func TestCheckPodRunningSuccessful(t *testing.T) { err := c.checkPodRunning("test_cmd") assert.Nil(t, err, "%v", err) } + +func TestContainerAddResources(t *testing.T) { + assert := assert.New(t) + + c := &Container{} + err := c.addResources() + assert.Nil(err) + + c.config = &ContainerConfig{Annotations: make(map[string]string)} + c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodSandbox) + err = c.addResources() + assert.Nil(err) + + c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodContainer) + err = c.addResources() + assert.Nil(err) + + c.config.Resources = ContainerResources{ + CPUQuota: 5000, + CPUPeriod: 1000, + } + c.pod = &Pod{hypervisor: &mockHypervisor{}} + err = c.addResources() + assert.Nil(err) +} + +func TestContainerRemoveResources(t *testing.T) { + assert := assert.New(t) + + c := &Container{} + err := c.addResources() + assert.Nil(err) + + c.config = &ContainerConfig{Annotations: make(map[string]string)} + c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodSandbox) + err = c.removeResources() + assert.Nil(err) + + c.config.Annotations[vcAnnotations.ContainerTypeKey] = string(PodContainer) + err = c.removeResources() + assert.Nil(err) + + c.config.Resources = ContainerResources{ + CPUQuota: 5000, + CPUPeriod: 1000, + } + c.pod = &Pod{hypervisor: &mockHypervisor{}} + err = c.removeResources() + assert.Nil(err) +} diff --git a/hyperstart_agent.go b/hyperstart_agent.go index 199ca3ad..29a73e7b 100644 --- a/hyperstart_agent.go +++ b/hyperstart_agent.go @@ -424,6 +424,13 @@ func (h *hyper) startOneContainer(pod Pod, c *Container) error { Process: process, } + if c.config.Resources.CPUQuota != 0 && c.config.Resources.CPUPeriod != 0 { + container.Constraints = hyperstart.Constraints{ + CPUQuota: c.config.Resources.CPUQuota, + CPUPeriod: c.config.Resources.CPUPeriod, + } + } + container.SystemMountsInfo.BindMountDev = c.systemMountsInfo.BindMountDev if c.state.Fstype != "" { diff --git a/hypervisor_test.go b/hypervisor_test.go index c6619f9d..a14892bc 100644 --- a/hypervisor_test.go +++ b/hypervisor_test.go @@ -181,6 +181,7 @@ func TestHypervisorConfigDefaults(t *testing.T) { DefaultMemSz: defaultMemSzMiB, DefaultBridges: defaultBridges, BlockDeviceDriver: defaultBlockDriver, + DefaultMaxVCPUs: defaultMaxQemuVCPUs, } if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false { t.Fatal() diff --git a/pkg/hyperstart/types.go b/pkg/hyperstart/types.go index 035f3d71..eeed10dc 100644 --- a/pkg/hyperstart/types.go +++ b/pkg/hyperstart/types.go @@ -202,6 +202,16 @@ type SystemMountsInfo struct { DevShmSize int `json:"devShmSize"` } +// Constraints describes the constrains for a container +type Constraints struct { + // CPUQuota specifies the total amount of time in microseconds + // The number of microseconds per CPUPeriod that the container is guaranteed CPU access + CPUQuota int64 + + // CPUPeriod specifies the CPU CFS scheduler period of time in microseconds + CPUPeriod uint64 +} + // Container describes a container running on a pod. type Container struct { ID string `json:"id"` @@ -216,6 +226,7 @@ type Container struct { RestartPolicy string `json:"restartPolicy"` Initialize bool `json:"initialize"` SystemMountsInfo SystemMountsInfo `json:"systemMountsInfo"` + Constraints Constraints `json:"constraints"` } // IPAddress describes an IP address and its network mask. diff --git a/pkg/oci/utils.go b/pkg/oci/utils.go index 383a4c3e..57d5750c 100644 --- a/pkg/oci/utils.go +++ b/pkg/oci/utils.go @@ -454,13 +454,7 @@ func vmConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.Resources, error) return vc.Resources{}, fmt.Errorf("Invalid OCI cpu period %d", period) } - // Use some math magic to round up to the nearest whole vCPU - // (that is, a partial part of a quota request ends up assigning - // a whole vCPU, for instance, a request of 1.5 'cpu quotas' - // will give 2 vCPUs). - // This also has the side effect that we will always allocate - // at least 1 vCPU. - resources.VCPUs = uint((uint64(quota) + (period - 1)) / period) + resources.VCPUs = vc.ConstraintsToVCPUs(quota, period) } return resources, nil @@ -587,6 +581,14 @@ func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, det return vc.ContainerConfig{}, err } + var resources vc.ContainerResources + if ocispec.Linux.Resources.CPU != nil && + ocispec.Linux.Resources.CPU.Quota != nil && + ocispec.Linux.Resources.CPU.Period != nil { + resources.CPUQuota = *ocispec.Linux.Resources.CPU.Quota + resources.CPUPeriod = *ocispec.Linux.Resources.CPU.Period + } + containerConfig := vc.ContainerConfig{ ID: cid, RootFs: rootfs, @@ -598,6 +600,7 @@ func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, det }, Mounts: containerMounts(ocispec), DeviceInfos: deviceInfos, + Resources: resources, } cType, err := ocispec.ContainerType() diff --git a/qemu_test.go b/qemu_test.go index ac907b4f..67defe4c 100644 --- a/qemu_test.go +++ b/qemu_test.go @@ -37,6 +37,7 @@ func newQemuConfig() HypervisorConfig { DefaultMemSz: defaultMemSzMiB, DefaultBridges: defaultBridges, BlockDeviceDriver: defaultBlockDriver, + DefaultMaxVCPUs: defaultMaxQemuVCPUs, } } diff --git a/utils.go b/utils.go index 7fcc66d1..69e63f58 100644 --- a/utils.go +++ b/utils.go @@ -96,3 +96,18 @@ func writeToFile(path string, data []byte) error { return nil } + +// ConstraintsToVCPUs converts CPU quota and period to vCPUs +func ConstraintsToVCPUs(quota int64, period uint64) uint { + if quota != 0 && period != 0 { + // Use some math magic to round up to the nearest whole vCPU + // (that is, a partial part of a quota request ends up assigning + // a whole vCPU, for instance, a request of 1.5 'cpu quotas' + // will give 2 vCPUs). + // This also has the side effect that we will always allocate + // at least 1 vCPU. + return uint((uint64(quota) + (period - 1)) / period) + } + + return 0 +} diff --git a/utils_test.go b/utils_test.go index 7dbe6951..dac0a1a4 100644 --- a/utils_test.go +++ b/utils_test.go @@ -162,3 +162,20 @@ func TestWriteToFile(t *testing.T) { assert.True(t, reflect.DeepEqual(testData, data)) } + +func TestConstraintsToVCPUs(t *testing.T) { + assert := assert.New(t) + + vcpus := ConstraintsToVCPUs(0, 100) + assert.Zero(vcpus) + + vcpus = ConstraintsToVCPUs(100, 0) + assert.Zero(vcpus) + + expectedVCPUs := uint(4) + vcpus = ConstraintsToVCPUs(4000, 1000) + assert.Equal(expectedVCPUs, vcpus) + + vcpus = ConstraintsToVCPUs(4000, 1200) + assert.Equal(expectedVCPUs, vcpus) +}