Skip to content

Commit 752703f

Browse files
AkihiroSudafahedouch
AkihiroSuda
and
fahedouch
committed
[WIP] [Carry 2535] rootless: support detach-netns mode
When RootlessKit v2.0 (rootless-containers/rootlesskit PR 379) is installed, `containerd-rootless.sh` launches it with `--detach-netns` so as to run the daemon in the host network namespace. This will enable: - Accelerated (and deflaked) `nerdctl pull`, `nerdctl push`, `nerdctl build`, etc - Proper support for `nerdctl pull 127.0.0.1:.../...` - Proper support for `nerdctl run --net=host` Replaces Fahed Dorgaa's PR 2535 Co-authored-by: fahed dorgaa <fahed.dorgaa@gmail.com> Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
1 parent f064081 commit 752703f

File tree

14 files changed

+213
-3
lines changed

14 files changed

+213
-3
lines changed

‎Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ ARG STARGZ_SNAPSHOTTER_VERSION=v0.15.1
2929
# Extra deps: Encryption
3030
ARG IMGCRYPT_VERSION=v1.1.9
3131
# Extra deps: Rootless
32-
ARG ROOTLESSKIT_VERSION=v1.1.1
32+
ARG ROOTLESSKIT_VERSION=v2.0.0-alpha.2
3333
ARG SLIRP4NETNS_VERSION=v1.2.2
3434
# Extra deps: bypass4netns
3535
ARG BYPASS4NETNS_VERSION=v0.3.0

‎Dockerfile.d/SHA256SUMS.d/rootlesskit-v2.0.0-alpha.2

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
7c8c07c159aef32b5e68f5b8dc508dc422499744f61fa916c246bfae16a0d85c rootlesskit-aarch64.tar.gz
2+
683ba2c34bfa4a3477f9c50508a233c94b71a6eeaa0ee080abd1ebc2c09a8b9c rootlesskit-armv7l.tar.gz
3+
844b6297021d914be9f266e341ff77da4b98f43074504fe9cc020ae5c61a791d rootlesskit-ppc64le.tar.gz
4+
d317e9c519d862508d6659083f9e1773573e899aa6e48e89d121211e5e823b6a rootlesskit-riscv64.tar.gz
5+
720b425c608f8ab4326354582bc92825031d9d8c40865df155c2c7cb8368f115 rootlesskit-s390x.tar.gz
6+
d29edd2e3d903974754edb14b251ef19bfa9317e6626436fac760d1213879e8d rootlesskit-x86_64.tar.gz

‎extras/rootless/containerd-rootless-setuptool.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,12 @@ cmd_entrypoint_check() {
143143
cmd_entrypoint_nsenter() {
144144
# No need to call init()
145145
pid=$(cat "$XDG_RUNTIME_DIR/containerd-rootless/child_pid")
146-
exec nsenter --no-fork --wd="$(pwd)" --preserve-credentials -m -n -U -t "$pid" -- "$@"
146+
n=""
147+
# If RootlessKit is running with `--detach-netns` mode, we do NOT enter the detached netns here
148+
if [ ! -e "$XDG_RUNTIME_DIR/containerd-rootless/netns" ]; then
149+
n="-n"
150+
fi
151+
exec nsenter --no-fork --wd="$(pwd)" --preserve-credentials -m $n -U -t "$pid" -- "$@"
147152
}
148153

149154
show_systemd_error() {

‎extras/rootless/containerd-rootless.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ if [ -z $_CONTAINERD_ROOTLESS_CHILD ]; then
107107
export _CONTAINERD_ROOTLESS_SELINUX
108108
fi
109109
fi
110+
if rootlesskit --help | grep -qw -- "--detach-netns"; then
111+
CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS=--detach-netns $CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS
112+
fi
110113
# Re-exec the script via RootlessKit, so as to create unprivileged {user,mount,network} namespaces.
111114
#
112115
# --copy-up allows removing/creating files in the directories by creating tmpfs and symlinks

‎pkg/cmd/container/create.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import (
5050
"github.com/containerd/nerdctl/pkg/namestore"
5151
"github.com/containerd/nerdctl/pkg/platformutil"
5252
"github.com/containerd/nerdctl/pkg/referenceutil"
53+
"github.com/containerd/nerdctl/pkg/rootlessutil"
5354
"github.com/containerd/nerdctl/pkg/strutil"
5455
dockercliopts "github.com/docker/cli/opts"
5556
dockeropts "github.com/docker/docker/opts"
@@ -418,6 +419,29 @@ func GenerateLogURI(dataStore string) (*url.URL, error) {
418419
}
419420

420421
func withNerdctlOCIHook(cmd string, args []string) (oci.SpecOpts, error) {
422+
if rootlessutil.IsRootless() {
423+
detachedNetNS, err := rootlessutil.DetachedNetNS()
424+
if err != nil {
425+
return nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
426+
}
427+
if detachedNetNS != "" {
428+
// Rewrite {cmd, args} if RootlessKit with running with --detach-netns, so that the hook can gain
429+
// CAP_NET_ADMIN in the namespaces.
430+
// - Old:
431+
// - cmd: "/usr/local/bin/nerdctl"
432+
// - args: {"--data-root=/foo"}
433+
// - New:
434+
// - cmd: "/usr/bin/nsenter"
435+
// - args: {"-n/run/user/1000/containerd-rootless/netns", "--", "/usr/local/bin/nerdctl", "--data-root=/foo"}
436+
oldCmd, oldArgs := cmd, args
437+
cmd, err = exec.LookPath("nsenter")
438+
if err != nil {
439+
return nil, err
440+
}
441+
args = append([]string{"-n" + detachedNetNS, "--", oldCmd}, oldArgs...)
442+
}
443+
}
444+
421445
args = append([]string{cmd}, append(args, "internal", "oci-hook")...)
422446
return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
423447
if s.Hooks == nil {

‎pkg/containerutil/container_network_manager.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
"github.com/containerd/nerdctl/pkg/mountutil"
3939
"github.com/containerd/nerdctl/pkg/netutil"
4040
"github.com/containerd/nerdctl/pkg/netutil/nettype"
41+
"github.com/containerd/nerdctl/pkg/rootlessutil"
4142
"github.com/containerd/nerdctl/pkg/strutil"
4243
"github.com/opencontainers/runtime-spec/specs-go"
4344
)
@@ -461,9 +462,37 @@ func (m *hostNetworkManager) ContainerNetworkingOpts(_ context.Context, containe
461462
}
462463
}
463464

465+
if rootlessutil.IsRootless() {
466+
detachedNetNS, err := rootlessutil.DetachedNetNS()
467+
if err != nil {
468+
return nil, nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
469+
}
470+
if detachedNetNS != "" {
471+
// For rootless + host netns, we can't mount /sys.
472+
//
473+
// TODO: consider to just bind-mount /sys from the host with rro,
474+
// when rro is available (kernel >= 5.12, runc >= 1.1)
475+
//
476+
// Relevant: https://github.com/moby/buildkit/blob/v0.12.4/util/rootless/specconv/specconv_linux.go#L15-L34
477+
specs = append(specs, withRemoveSysfs)
478+
}
479+
}
480+
464481
return specs, cOpts, nil
465482
}
466483

484+
func withRemoveSysfs(_ context.Context, _ oci.Client, c *containers.Container, s *oci.Spec) error {
485+
var mounts []specs.Mount
486+
for _, mount := range s.Mounts {
487+
if strings.HasPrefix(mount.Destination, "/sys") {
488+
continue
489+
}
490+
mounts = append(mounts, mount)
491+
}
492+
s.Mounts = mounts
493+
return nil
494+
}
495+
467496
// types.NetworkOptionsManager implementation for CNI networking settings.
468497
// This is a more specialized and OS-dependendant networking model so this
469498
// struct provides different implementations on different platforms.

‎pkg/containerutil/container_network_manager_linux.go

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@ package containerutil
1919
import (
2020
"context"
2121
"errors"
22+
"fmt"
2223
"io/fs"
2324
"path/filepath"
2425

2526
"github.com/containerd/containerd"
27+
"github.com/containerd/containerd/containers"
2628
"github.com/containerd/containerd/oci"
2729
"github.com/containerd/log"
2830
"github.com/containerd/nerdctl/pkg/api/types"
@@ -32,6 +34,7 @@ import (
3234
"github.com/containerd/nerdctl/pkg/netutil"
3335
"github.com/containerd/nerdctl/pkg/resolvconf"
3436
"github.com/containerd/nerdctl/pkg/rootlessutil"
37+
"github.com/opencontainers/runtime-spec/specs-go"
3538
)
3639

3740
type cniNetworkManagerPlatform struct {
@@ -124,6 +127,16 @@ func (m *cniNetworkManager) ContainerNetworkingOpts(_ context.Context, container
124127
}
125128
}
126129

130+
if rootlessutil.IsRootless() {
131+
detachedNetNS, err := rootlessutil.DetachedNetNS()
132+
if err != nil {
133+
return nil, nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
134+
}
135+
if detachedNetNS != "" {
136+
opts = append(opts, withRootlessKitDetachedNetNS(detachedNetNS, stateDir))
137+
}
138+
}
139+
127140
return opts, cOpts, nil
128141
}
129142

@@ -172,3 +185,23 @@ func (m *cniNetworkManager) buildResolvConf(resolvConfPath string) error {
172185
_, err = resolvconf.Build(resolvConfPath, append(slirp4Dns, nameServers...), searchDomains, dnsOptions)
173186
return err
174187
}
188+
189+
// withRootlessKitDetachedNetNS is used when all the following conditions are satisfied:
190+
// - Rootless mode
191+
// - RootlessKit is running with detach-netns mode (available since RootlessKit v2.0)
192+
// - the network manager is not configured to none/host/container mode
193+
func withRootlessKitDetachedNetNS(detachedNetNS, containerStateDir string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error {
194+
return func(_ context.Context, _ oci.Client, c *containers.Container, s *oci.Spec) error {
195+
for i := range s.Linux.Namespaces {
196+
n := &s.Linux.Namespaces[i]
197+
if n.Type == specs.NetworkNamespace && n.Path == "" {
198+
newChildNetNS := filepath.Join(containerStateDir, "netns")
199+
if err := netutil.NewNestedNetNS(detachedNetNS, newChildNetNS); err != nil {
200+
return fmt.Errorf("failed to nest a new netns %q inside %q: %w", newChildNetNS, detachedNetNS, err)
201+
}
202+
n.Path = newChildNetNS
203+
}
204+
}
205+
return nil
206+
}
207+
}

‎pkg/netutil/netutil_linux.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package netutil
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"os/exec"
23+
)
24+
25+
// NewNestedNetNS creates a new netns that is nested in the parent.
26+
func NewNestedNetNS(parent, newChild string) error {
27+
if err := os.WriteFile(newChild, nil, 0400); err != nil {
28+
return err
29+
}
30+
// this is hard (not impossible though) to reimplement in Go: https://github.com/cloudflare/slirpnetstack/commit/d7766a8a77f0093d3cb7a94bd0ccbe3f67d411ba
31+
cmd := exec.Command("nsenter", "-n"+parent, "--",
32+
"unshare", "-n", "--", "mount", "--bind", "/proc/self/ns/net", newChild)
33+
out, err := cmd.CombinedOutput()
34+
if err != nil {
35+
return fmt.Errorf("failed to execute %v: %w (out=%q)", cmd.Args, err, string(out))
36+
}
37+
return nil
38+
}

‎pkg/ocihook/ocihook.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,11 @@ func onPostStop(opts *handlerOpts) error {
518518
log.L.WithError(err).Errorf("failed to call cni.Remove")
519519
return err
520520
}
521+
if rootlessutil.IsRootlessChild() {
522+
if err := unmountNetNSNestedInsideDetachedNetNS(ctx, opts.state.Annotations[labels.StateDir]); err != nil {
523+
log.L.WithError(err).Warn("failed to unmount the netns that was nested in RootlessKit's detach-netns")
524+
}
525+
}
521526
hs, err := hostsstore.NewStore(opts.dataStore)
522527
if err != nil {
523528
return err

‎pkg/ocihook/rootless_linux.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,14 @@ package ocihook
1818

1919
import (
2020
"context"
21+
"errors"
22+
"os"
23+
"path/filepath"
2124

2225
gocni "github.com/containerd/go-cni"
2326
"github.com/containerd/nerdctl/pkg/rootlessutil"
2427
rlkclient "github.com/rootless-containers/rootlesskit/pkg/api/client"
28+
"golang.org/x/sys/unix"
2529
)
2630

2731
func exposePortsRootless(ctx context.Context, rlkClient rlkclient.Client, ports []gocni.PortMapping) error {
@@ -51,3 +55,20 @@ func unexposePortsRootless(ctx context.Context, rlkClient rlkclient.Client, port
5155

5256
return nil
5357
}
58+
59+
// unmountNetNSNestedInsideDetachedNetNS unmounts the netns that was nested inside
60+
// RootlessKit's detach-netns
61+
func unmountNetNSNestedInsideDetachedNetNS(ctx context.Context, dir string) error {
62+
if dir == "" {
63+
return nil
64+
}
65+
netnsPath := filepath.Join(dir, "netns")
66+
if _, err := os.Stat(netnsPath); err != nil {
67+
if errors.Is(err, os.ErrNotExist) {
68+
return nil
69+
}
70+
return err
71+
}
72+
// "detach" in umount(2)'s lingo here has nothing to do with "detach" in RootlessKit's lingo
73+
return unix.Unmount(netnsPath, unix.MNT_DETACH)
74+
}

0 commit comments

Comments
 (0)