Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,7 @@ docker-tests: integration-test-images $(RUNTIME_BIN)
@$(call install_runtime_noreload,$(RUNTIME)-docker,--net-raw --allow-packet-socket-write) # Used by TestDocker*.
@$(call install_runtime_noreload,$(RUNTIME)-fdlimit,--fdlimit=2000) # Used by TestRlimitNoFile.
@$(call install_runtime_noreload,$(RUNTIME)-dcache,--fdlimit=2000 --dcache=100) # Used by TestDentryCacheLimit.
@$(call install_runtime_noreload,$(RUNTIME)-cpunumfixed,--cpu-num-fixed=8) # Used by TestNumCPUFixed.
@$(call install_runtime_noreload,$(RUNTIME)-host-uds,--host-uds=all) # Used by TestHostSocketConnect.
@$(call install_runtime_noreload,$(RUNTIME)-overlay,--overlay2=all:self) # Used by TestOverlay*.
@$(call install_runtime,$(RUNTIME)-cgroupv2,--mount-cgroup-v2) # Used by TestSystemd* and TestPIDFDSelftests.
Expand Down
9 changes: 9 additions & 0 deletions runsc/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,12 @@ type Config struct {
// E.g. 0.2 CPU quota will result in 1, and 1.9 in 2.
CPUNumFromQuota bool `flag:"cpu-num-from-quota"`

// CPUNumFixed, when > 0, exposes exactly this many CPUs to the sandbox,
// independent of the cgroup quota and host CPU count, so it can boot with
// headroom for a later in-place quota increase. May exceed the host count;
// the quota still bounds real CPU. Takes precedence over CPUNumFromQuota.
CPUNumFixed int `flag:"cpu-num-fixed"`

// Allows overriding of flags in OCI annotations.
AllowFlagOverride bool `flag:"allow-flag-override"`

Expand Down Expand Up @@ -495,6 +501,9 @@ func (c *Config) Validate() error {
if c.NumNetworkChannels <= 0 {
return fmt.Errorf("num_network_channels must be > 0, got: %d", c.NumNetworkChannels)
}
if c.CPUNumFixed < 0 {
return fmt.Errorf("cpu-num-fixed must be >= 0, got: %d", c.CPUNumFixed)
}
if c.PauseExternalNetworking && c.Network != NetworkSandbox {
return fmt.Errorf("pause-external-networking flag is only supported with sandbox networking")
}
Expand Down
1 change: 1 addition & 0 deletions runsc/config/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
flagSet.Bool("rootless", false, "it allows the sandbox to be started with a user that is not root. Sandbox and Gofer processes may run with same privileges as current user.")
flagSet.Var(leakModePtr(refs.NoLeakChecking), "ref-leak-mode", "sets reference leak check mode: disabled (default), log-names, log-traces.")
flagSet.Bool("cpu-num-from-quota", true, "set cpu number to cpu quota (least integer greater or equal to quota value, but not less than 2)")
flagSet.Int("cpu-num-fixed", 0, "if > 0, expose exactly this many CPUs to the sandbox regardless of cgroup quota and host CPU count; takes precedence over --cpu-num-from-quota.")
flagSet.Bool(flagOCISeccomp, false, "Enables loading OCI seccomp filters inside the sandbox.")
flagSet.Bool("enable-core-tags", false, "enables core tagging. Requires host linux kernel >= 5.14.")
flagSet.String("pod-init-config", "", "path to configuration file with additional steps to take during pod creation.")
Expand Down
1 change: 1 addition & 0 deletions runsc/sandbox/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ go_test(
name = "sandbox_test",
size = "small",
srcs = [
"cpu_test.go",
"network_test.go",
],
library = ":sandbox",
Expand Down
117 changes: 117 additions & 0 deletions runsc/sandbox/cpu_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright 2026 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sandbox

import (
"testing"

"gvisor.dev/gvisor/runsc/config"
)

func TestCPUNumForSandbox(t *testing.T) {
const period = 100000 // 100ms, the usual CFS period.
for _, tc := range []struct {
name string
fixed int
fromQuota bool
cpuNum int
quota int64
want int
}{
{
name: "fixed overrides quota and may exceed host cpus",
fixed: 16,
fromQuota: true,
cpuNum: 8,
quota: 1 * period,
want: 16,
},
{
name: "fixed overrides an unlimited quota",
fixed: 8,
fromQuota: true,
cpuNum: 4,
quota: 0,
want: 8,
},
{
name: "from quota rounds up",
fromQuota: true,
cpuNum: 8,
quota: 150000, // 1.5 CPUs
want: 2,
},
{
name: "from quota floors at two cpus",
fromQuota: true,
cpuNum: 8,
quota: 50000, // 0.5 CPUs
want: 2,
},
{
name: "from quota only lowers, never raises",
fromQuota: true,
cpuNum: 8,
quota: 16 * period,
want: 8,
},
{
name: "from quota disabled leaves cpu count unchanged",
fromQuota: false,
cpuNum: 8,
quota: 1 * period,
want: 8,
},
{
name: "unlimited quota leaves cpu count unchanged",
fromQuota: true,
cpuNum: 8,
quota: 0,
want: 8,
},
{
name: "no cgroup, no fixed count defaults to host (zero)",
fromQuota: true,
cpuNum: 0,
quota: 0,
want: 0,
},
{
name: "fixed count with no cgroup",
fixed: 8,
cpuNum: 0,
quota: 0,
want: 8,
},
{
name: "fixed count wins with from-quota disabled",
fixed: 8,
cpuNum: 4,
quota: 16 * period,
want: 8,
},
} {
t.Run(tc.name, func(t *testing.T) {
conf := &config.Config{
CPUNumFixed: tc.fixed,
CPUNumFromQuota: tc.fromQuota,
}
if got := cpuNumForSandbox(conf, tc.cpuNum, tc.quota, period); got != tc.want {
t.Errorf("cpuNumForSandbox(fixed=%d, fromQuota=%t, cpuNum=%d, quota=%d) = %d, want %d",
tc.fixed, tc.fromQuota, tc.cpuNum, tc.quota, got, tc.want)
}
})
}
}
61 changes: 37 additions & 24 deletions runsc/sandbox/sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -879,6 +879,30 @@ func (s *Sandbox) connError(err error) error {
return fmt.Errorf("connecting to control server at PID %d: %v", s.Pid.Load(), err)
}

// cpuNumForSandbox returns the CPU count to expose to the sandbox, or 0 to use
// boot's host default. CPUNumFixed pins it independent of the quota and host
// count (may exceed the host; the quota still caps real CPU); otherwise
// CPUNumFromQuota lowers cpuNum to the quota rounded up, min two.
func cpuNumForSandbox(conf *config.Config, cpuNum int, cpuQuota, cpuPeriod int64) int {
if conf.CPUNumFixed > 0 {
return conf.CPUNumFixed
}
if conf.CPUNumFromQuota && cpuQuota > 0 && cpuPeriod > 0 {
const minCPUs = 2
quota := float64(cpuQuota) / float64(cpuPeriod)
if n := int(math.Ceil(quota)); n > 0 {
if n < minCPUs {
n = minCPUs
}
if n < cpuNum {
// Only lower the cpu number.
cpuNum = n
}
}
}
return cpuNum
}

// createSandboxProcess starts the sandbox as a subprocess by running the "boot"
// command, passing in the bundle dir.
func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyncFile *os.File) error {
Expand Down Expand Up @@ -1330,37 +1354,23 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
cmd.Args = append(cmd.Args, "--total-host-memory", strconv.FormatUint(totalSysMem, 10))

mem := totalSysMem
// cgroupCPUNum is 0 when no host cgroup is present (sentry-managed cgroups):
// cpuNumForSandbox then honors CPUNumFixed and otherwise returns 0, which
// leaves the count to boot's host default. The quota derivation only runs
// when the cgroup (kubelet- or sentry-managed) provides quota and period.
cgroupCPUNum := 0
var cpuQuota, cpuPeriod int64
if s.CgroupJSON.Cgroup != nil {
cpuNum, err := s.CgroupJSON.Cgroup.NumCPU()
if err != nil {
var err error
if cgroupCPUNum, err = s.CgroupJSON.Cgroup.NumCPU(); err != nil {
return fmt.Errorf("getting cpu count from cgroups: %v", err)
}
cpuQuota, err := s.CgroupJSON.Cgroup.CPUQuota()
if err != nil {
if cpuQuota, err = s.CgroupJSON.Cgroup.CPUQuota(); err != nil {
return fmt.Errorf("getting raw cpu quota from cgroups: %v", err)
}
cpuPeriod, err := s.CgroupJSON.Cgroup.CPUPeriod()
if err != nil {
if cpuPeriod, err = s.CgroupJSON.Cgroup.CPUPeriod(); err != nil {
return fmt.Errorf("getting raw cpu period from cgroups: %v", err)
}
if conf.CPUNumFromQuota && cpuQuota > 0 && cpuPeriod > 0 {
// Dropping below 2 CPUs can trigger application to disable
// locks that can lead do hard to debug errors, so just
// leaving two cores as reasonable default.
const minCPUs = 2

quota := float64(cpuQuota) / float64(cpuPeriod)
if n := int(math.Ceil(quota)); n > 0 {
if n < minCPUs {
n = minCPUs
}
if n < cpuNum {
// Only lower the cpu number.
cpuNum = n
}
}
}
cmd.Args = append(cmd.Args, "--cpu-num", strconv.Itoa(cpuNum))
if cpuQuota > 0 {
cmd.Args = append(cmd.Args, "--cpu-quota", strconv.FormatInt(cpuQuota, 10))
}
Expand All @@ -1376,6 +1386,9 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
mem = memLimit
}
}
if cpuNum := cpuNumForSandbox(conf, cgroupCPUNum, cpuQuota, cpuPeriod); cpuNum > 0 {
cmd.Args = append(cmd.Args, "--cpu-num", strconv.Itoa(cpuNum))
}
cmd.Args = append(cmd.Args, "--total-memory", strconv.FormatUint(mem, 10))

if args.Attached {
Expand Down
29 changes: 29 additions & 0 deletions test/e2e/integration_runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,35 @@ func TestRlimitNoFile(t *testing.T) {
}
}

// cpuNumFixed is the value the -cpunumfixed runtime is installed with in the
// Makefile (--cpu-num-fixed). Keep the two in sync.
const cpuNumFixed = 8

// TestNumCPUFixed checks that --cpu-num-fixed pins the number of CPUs the
// sandbox sees, independent of the cgroup cpuset and quota.
func TestNumCPUFixed(t *testing.T) {
ctx := context.Background()
d := dockerutil.MakeContainerWithRuntime(ctx, t, "-cpunumfixed")
defer d.CleanUp(ctx)

// Pin the container to a single host CPU; the sandbox must still report the
// fixed count rather than one.
out, err := d.Run(ctx, dockerutil.RunOpts{
Image: "basic/alpine",
CpusetCpus: "0",
}, "sh", "-c", "cat /proc/cpuinfo | grep 'processor.*:' | wc -l")
if err != nil {
t.Fatalf("docker run failed: %v", err)
}
got, err := strconv.Atoi(strings.TrimSpace(out))
if err != nil {
t.Fatalf("failed to parse %q: %v", out, err)
}
if got != cpuNumFixed {
t.Errorf("NumCPU got: %d, want: %d", got, cpuNumFixed)
}
}

func TestDentryCacheLimit(t *testing.T) {
ctx := context.Background()
d := dockerutil.MakeContainerWithRuntime(ctx, t, "-dcache")
Expand Down