Skip to content

Commit

Permalink
Merge pull request moby#41182 from cpuguy83/runtime_configure_shim
Browse files Browse the repository at this point in the history
  • Loading branch information
cpuguy83 authored Jul 14, 2020
2 parents 3f36764 + f63f73a commit 61b73ee
Show file tree
Hide file tree
Showing 18 changed files with 238 additions and 178 deletions.
10 changes: 10 additions & 0 deletions api/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,16 @@ type Checkpoint struct {
type Runtime struct {
Path string `json:"path"`
Args []string `json:"runtimeArgs,omitempty"`

// This is exposed here only for internal use
// It is not currently supported to specify custom shim configs
Shim *ShimConfig `json:"-"`
}

// ShimConfig is used by runtime to configure containerd shims
type ShimConfig struct {
Binary string
Opts interface{}
}

// DiskUsage contains response of Engine API:
Expand Down
29 changes: 22 additions & 7 deletions daemon/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,6 @@ const (
// maximum number of attempts that
// may take place at a time for each pull when the connection is lost.
DefaultDownloadAttempts = 5
// StockRuntimeName is the reserved name/alias used to represent the
// OCI runtime being shipped with the docker daemon package.
StockRuntimeName = "runc"
// DefaultShmSize is the default value for container's shm size
DefaultShmSize = int64(67108864)
// DefaultNetworkMtu is the default value for network MTU
Expand All @@ -47,8 +44,24 @@ const (
DisableNetworkBridge = "none"
// DefaultInitBinary is the name of the default init binary
DefaultInitBinary = "docker-init"

// StockRuntimeName is the reserved name/alias used to represent the
// OCI runtime being shipped with the docker daemon package.
StockRuntimeName = "runc"
// LinuxV1RuntimeName is the runtime used to specify the containerd v1 shim with the runc binary
// Note this is different than io.containerd.runc.v1 which would be the v1 shim using the v2 shim API.
// This is specifically for the v1 shim using the v1 shim API.
LinuxV1RuntimeName = "io.containerd.runtime.v1.linux"
// LinuxV2RuntimeName is the runtime used to specify the containerd v2 runc shim
LinuxV2RuntimeName = "io.containerd.runc.v2"
)

var builtinRuntimes = map[string]bool{
StockRuntimeName: true,
LinuxV1RuntimeName: true,
LinuxV2RuntimeName: true,
}

// flatOptions contains configuration keys
// that MUST NOT be parsed as deep structures.
// Use this to differentiate these options
Expand Down Expand Up @@ -571,10 +584,12 @@ func Validate(config *Config) error {
return err
}

if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" && defaultRuntime != StockRuntimeName {
runtimes := config.GetAllRuntimes()
if _, ok := runtimes[defaultRuntime]; !ok {
return fmt.Errorf("specified default runtime '%s' does not exist", defaultRuntime)
if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" {
if !builtinRuntimes[defaultRuntime] {
runtimes := config.GetAllRuntimes()
if _, ok := runtimes[defaultRuntime]; !ok {
return fmt.Errorf("specified default runtime '%s' does not exist", defaultRuntime)
}
}
}

Expand Down
8 changes: 6 additions & 2 deletions daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -932,7 +932,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
}
}

return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, d.useShimV2())
var rt types.Runtime
if runtime := config.GetRuntime(config.GetDefaultRuntimeName()); runtime != nil {
rt = *runtime
}
return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, rt)
}

// Plugin system initialization should happen before restore. Do not change order.
Expand Down Expand Up @@ -1081,7 +1085,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S

go d.execCommandGC()

d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d, d.useShimV2())
d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d)
if err != nil {
return nil, err
}
Expand Down
74 changes: 11 additions & 63 deletions daemon/daemon_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"github.com/docker/docker/opts"
"github.com/docker/docker/pkg/containerfs"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/parsers/kernel"
"github.com/docker/docker/pkg/sysinfo"
Expand Down Expand Up @@ -78,10 +77,6 @@ const (
cgroupFsDriver = "cgroupfs"
cgroupSystemdDriver = "systemd"
cgroupNoneDriver = "none"

// DefaultRuntimeName is the default runtime to be used by
// containerd if none is specified
DefaultRuntimeName = "runc"
)

type containerGetter interface {
Expand Down Expand Up @@ -729,55 +724,11 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.
}
}

return warnings, nil
}

func (daemon *Daemon) loadRuntimes() error {
return daemon.initRuntimes(daemon.configStore.Runtimes)
}

func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error) {
runtimeDir := filepath.Join(daemon.configStore.Root, "runtimes")
// Remove old temp directory if any
os.RemoveAll(runtimeDir + "-old")
tmpDir, err := ioutils.TempDir(daemon.configStore.Root, "gen-runtimes")
if err != nil {
return errors.Wrap(err, "failed to get temp dir to generate runtime scripts")
if hostConfig.Runtime == config.LinuxV1RuntimeName || (hostConfig.Runtime == "" && daemon.configStore.DefaultRuntime == config.LinuxV1RuntimeName) {
warnings = append(warnings, fmt.Sprintf("Configured runtime %q is deprecated and will be removed in the next release.", config.LinuxV1RuntimeName))
}
defer func() {
if err != nil {
if err1 := os.RemoveAll(tmpDir); err1 != nil {
logrus.WithError(err1).WithField("dir", tmpDir).
Warn("failed to remove tmp dir")
}
return
}

if err = os.Rename(runtimeDir, runtimeDir+"-old"); err != nil {
return
}
if err = os.Rename(tmpDir, runtimeDir); err != nil {
err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start")
return
}
if err = os.RemoveAll(runtimeDir + "-old"); err != nil {
logrus.WithError(err).WithField("dir", tmpDir).
Warn("failed to remove old runtimes dir")
}
}()

for name, rt := range runtimes {
if len(rt.Args) == 0 {
continue
}

script := filepath.Join(tmpDir, name)
content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
if err := ioutil.WriteFile(script, []byte(content), 0700); err != nil {
return err
}
}
return nil
return warnings, nil
}

// verifyDaemonSettings performs validation of daemon config struct
Expand Down Expand Up @@ -808,14 +759,15 @@ func verifyDaemonSettings(conf *config.Config) error {
return fmt.Errorf("exec-opt native.cgroupdriver=systemd requires cgroup v2 for rootless mode")
}

if conf.DefaultRuntime == "" {
conf.DefaultRuntime = config.StockRuntimeName
}
if conf.Runtimes == nil {
conf.Runtimes = make(map[string]types.Runtime)
configureRuntimes(conf)
if rtName := conf.GetDefaultRuntimeName(); rtName != "" {
if conf.GetRuntime(rtName) == nil {
return fmt.Errorf("specified default runtime '%s' does not exist", rtName)
}
if rtName == config.LinuxV1RuntimeName {
logrus.Warnf("Configured default runtime %q is deprecated and will be removed in the next release.", config.LinuxV1RuntimeName)
}
}
conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeName}

return nil
}

Expand Down Expand Up @@ -1756,10 +1708,6 @@ func (daemon *Daemon) setupSeccompProfile() error {
return nil
}

func (daemon *Daemon) useShimV2() bool {
return cgroups.IsCgroup2UnifiedMode()
}

// RawSysInfo returns *sysinfo.SysInfo .
func (daemon *Daemon) RawSysInfo(quiet bool) *sysinfo.SysInfo {
var opts []sysinfo.Opt
Expand Down
5 changes: 5 additions & 0 deletions daemon/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/docker/docker/api"
"github.com/docker/docker/api/types"
"github.com/docker/docker/cli/debug"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/dockerversion"
"github.com/docker/docker/pkg/fileutils"
Expand Down Expand Up @@ -78,6 +79,10 @@ func (daemon *Daemon) SystemInfo() *types.Info {
daemon.fillSecurityOptions(v, sysInfo)
daemon.fillLicense(v)

if v.DefaultRuntime == config.LinuxV1RuntimeName {
v.Warnings = append(v.Warnings, fmt.Sprintf("Configured default runtime %q is deprecated and will be removed in the next release.", config.LinuxV1RuntimeName))
}

return v
}

Expand Down
2 changes: 1 addition & 1 deletion daemon/reload_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]
if runtimeList.Len() > 0 {
runtimeList.WriteRune(' ')
}
runtimeList.WriteString(fmt.Sprintf("%s:%s", name, rt))
runtimeList.WriteString(fmt.Sprintf("%s:%s", name, rt.Path))
}

attributes["runtimes"] = runtimeList.String()
Expand Down
134 changes: 134 additions & 0 deletions daemon/runtime_unix.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
// +build !windows

package daemon

import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"

"github.com/containerd/containerd/runtime/linux/runctypes"
v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
"github.com/docker/docker/api/types"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/pkg/ioutils"
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

const (
defaultRuntimeName = "runc"

linuxShimV1 = "io.containerd.runtime.v1.linux"
linuxShimV2 = "io.containerd.runc.v2"
)

func configureRuntimes(conf *config.Config) {
if conf.DefaultRuntime == "" {
conf.DefaultRuntime = config.StockRuntimeName
}
if conf.Runtimes == nil {
conf.Runtimes = make(map[string]types.Runtime)
}
conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: defaultRuntimeName, Shim: getShimConfig(conf, defaultRuntimeName)}
conf.Runtimes[config.LinuxV1RuntimeName] = types.Runtime{Path: defaultRuntimeName, Shim: defaultV1ShimConfig(conf, defaultRuntimeName)}
conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName, Shim: defaultV2ShimConfig(conf, defaultRuntimeName)}
}

func getShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
if cgroups.IsCgroup2UnifiedMode() {
return defaultV2ShimConfig(conf, runtimePath)
}
return defaultV1ShimConfig(conf, runtimePath)
}

func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
return &types.ShimConfig{
Binary: linuxShimV2,
Opts: &v2runcoptions.Options{
BinaryName: runtimePath,
Root: filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
SystemdCgroup: UsingSystemd(conf),
NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
},
}
}

func defaultV1ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
return &types.ShimConfig{
Binary: linuxShimV1,
Opts: &runctypes.RuncOptions{
Runtime: runtimePath,
RuntimeRoot: filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
SystemdCgroup: UsingSystemd(conf),
},
}
}

func (daemon *Daemon) loadRuntimes() error {
return daemon.initRuntimes(daemon.configStore.Runtimes)
}

func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error) {
runtimeDir := filepath.Join(daemon.configStore.Root, "runtimes")
// Remove old temp directory if any
os.RemoveAll(runtimeDir + "-old")
tmpDir, err := ioutils.TempDir(daemon.configStore.Root, "gen-runtimes")
if err != nil {
return errors.Wrap(err, "failed to get temp dir to generate runtime scripts")
}
defer func() {
if err != nil {
if err1 := os.RemoveAll(tmpDir); err1 != nil {
logrus.WithError(err1).WithField("dir", tmpDir).
Warn("failed to remove tmp dir")
}
return
}

if err = os.Rename(runtimeDir, runtimeDir+"-old"); err != nil {
return
}
if err = os.Rename(tmpDir, runtimeDir); err != nil {
err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start")
return
}
if err = os.RemoveAll(runtimeDir + "-old"); err != nil {
logrus.WithError(err).WithField("dir", tmpDir).
Warn("failed to remove old runtimes dir")
}
}()

for name, rt := range runtimes {
if len(rt.Args) == 0 {
continue
}

script := filepath.Join(tmpDir, name)
content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
if err := ioutil.WriteFile(script, []byte(content), 0700); err != nil {
return err
}
}
return nil
}

// rewriteRuntimePath is used for runtimes which have custom arguments supplied.
// This is needed because the containerd API only calls the OCI runtime binary, there is no options for extra arguments.
// To support this case, the daemon wraps the specified runtime in a script that passes through those arguments.
func (daemon *Daemon) rewriteRuntimePath(name, p string, args []string) (string, error) {
if len(args) == 0 {
return p, nil
}

// Check that the runtime path actually exists here so that we can return a well known error.
if _, err := exec.LookPath(p); err != nil {
return "", errors.Wrap(err, "error while looking up the specified runtime path")
}

return filepath.Join(daemon.configStore.Root, "runtimes", name), nil
}
6 changes: 3 additions & 3 deletions daemon/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
}
}

createOptions, err := daemon.getLibcontainerdCreateOptions(container)
shim, createOptions, err := daemon.getLibcontainerdCreateOptions(container)
if err != nil {
return err
}
Expand All @@ -187,7 +187,7 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
return err
}

err = daemon.containerd.Create(ctx, container.ID, spec, createOptions, withImageName(imageRef.String()))
err = daemon.containerd.Create(ctx, container.ID, spec, shim, createOptions, withImageName(imageRef.String()))
if err != nil {
if errdefs.IsConflict(err) {
logrus.WithError(err).WithField("container", container.ID).Error("Container not cleaned up from containerd from previous run")
Expand All @@ -196,7 +196,7 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
if err := daemon.containerd.Delete(ctx, container.ID); err != nil && !errdefs.IsNotFound(err) {
logrus.WithError(err).WithField("container", container.ID).Error("Error cleaning up stale containerd container object")
}
err = daemon.containerd.Create(ctx, container.ID, spec, createOptions, withImageName(imageRef.String()))
err = daemon.containerd.Create(ctx, container.ID, spec, shim, createOptions, withImageName(imageRef.String()))
}
if err != nil {
return translateContainerdStartErr(container.Path, container.SetExitCode, err)
Expand Down
Loading

0 comments on commit 61b73ee

Please sign in to comment.