[WIP] [Carry 2535] rootless: support detach-netns mode

When RootlessKit v2.0 (rootless-containers/rootlesskit PR 379) is
installed, `containerd-rootless.sh` launches it with `--detach-netns`
so as to run the daemon in the host network namespace.

This will enable:
- Accelerated (and deflaked) `nerdctl pull`, `nerdctl push`, `nerdctl build`, etc
- Proper support for `nerdctl pull 127.0.0.1:.../...`
- Proper support for `nerdctl run --net=host`

Replaces Fahed Dorgaa's PR 2535

Co-authored-by: fahed dorgaa <fahed.dorgaa@gmail.com>
Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

Author: AkihiroSuda

Dockerfile

@@ -29,7 +29,7 @@ ARG STARGZ_SNAPSHOTTER_VERSION=v0.15.1
 # Extra deps: Encryption
 ARG IMGCRYPT_VERSION=v1.1.9
 # Extra deps: Rootless
-ARG ROOTLESSKIT_VERSION=v1.1.1
+ARG ROOTLESSKIT_VERSION=v2.0.0-alpha.2
 ARG SLIRP4NETNS_VERSION=v1.2.2
 # Extra deps: bypass4netns
 ARG BYPASS4NETNS_VERSION=v0.3.0

Dockerfile.d/SHA256SUMS.d/rootlesskit-v2.0.0-alpha.2

@@ -0,0 +1,6 @@
+7c8c07c159aef32b5e68f5b8dc508dc422499744f61fa916c246bfae16a0d85c  rootlesskit-aarch64.tar.gz
+683ba2c34bfa4a3477f9c50508a233c94b71a6eeaa0ee080abd1ebc2c09a8b9c  rootlesskit-armv7l.tar.gz
+844b6297021d914be9f266e341ff77da4b98f43074504fe9cc020ae5c61a791d  rootlesskit-ppc64le.tar.gz
+d317e9c519d862508d6659083f9e1773573e899aa6e48e89d121211e5e823b6a  rootlesskit-riscv64.tar.gz
+720b425c608f8ab4326354582bc92825031d9d8c40865df155c2c7cb8368f115  rootlesskit-s390x.tar.gz
+d29edd2e3d903974754edb14b251ef19bfa9317e6626436fac760d1213879e8d  rootlesskit-x86_64.tar.gz

extras/rootless/containerd-rootless-setuptool.sh

@@ -143,7 +143,12 @@ cmd_entrypoint_check() {
 cmd_entrypoint_nsenter() {
 	# No need to call init()
 	pid=$(cat "$XDG_RUNTIME_DIR/containerd-rootless/child_pid")
-	exec nsenter --no-fork --wd="$(pwd)" --preserve-credentials -m -n -U -t "$pid" -- "$@"
+	n=""
+	# If RootlessKit is running with `--detach-netns` mode, we do NOT enter the detached netns here
+	if [ ! -e "$XDG_RUNTIME_DIR/containerd-rootless/netns" ]; then
+		n="-n"
+	fi
+	exec nsenter --no-fork --wd="$(pwd)" --preserve-credentials -m $n -U -t "$pid" -- "$@"
 }
 
 show_systemd_error() {

extras/rootless/containerd-rootless.sh

@@ -107,6 +107,9 @@ if [ -z $_CONTAINERD_ROOTLESS_CHILD ]; then
 			export _CONTAINERD_ROOTLESS_SELINUX
 		fi
 	fi
+	if rootlesskit --help | grep -qw -- "--detach-netns"; then
+		CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS=--detach-netns $CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS
+	fi
 	# Re-exec the script via RootlessKit, so as to create unprivileged {user,mount,network} namespaces.
 	#
 	# --copy-up allows removing/creating files in the directories by creating tmpfs and symlinks

pkg/cmd/container/create.go

@@ -50,6 +50,7 @@ import (
 	"github.com/containerd/nerdctl/pkg/namestore"
 	"github.com/containerd/nerdctl/pkg/platformutil"
 	"github.com/containerd/nerdctl/pkg/referenceutil"
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
 	dockercliopts "github.com/docker/cli/opts"
 	dockeropts "github.com/docker/docker/opts"
@@ -418,6 +419,29 @@ func GenerateLogURI(dataStore string) (*url.URL, error) {
 }
 
 func withNerdctlOCIHook(cmd string, args []string) (oci.SpecOpts, error) {
+	if rootlessutil.IsRootless() {
+		detachedNetNS, err := rootlessutil.DetachedNetNS()
+		if err != nil {
+			return nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
+		}
+		if detachedNetNS != "" {
+			// Rewrite {cmd, args} if RootlessKit with running with --detach-netns, so that the hook can gain
+			// CAP_NET_ADMIN in the namespaces.
+			//   - Old:
+			//     - cmd:  "/usr/local/bin/nerdctl"
+			//     - args: {"--data-root=/foo"}
+			//   - New:
+			//     - cmd:  "/usr/bin/nsenter"
+			//     - args: {"-n/run/user/1000/containerd-rootless/netns", "--", "/usr/local/bin/nerdctl", "--data-root=/foo"}
+			oldCmd, oldArgs := cmd, args
+			cmd, err = exec.LookPath("nsenter")
+			if err != nil {
+				return nil, err
+			}
+			args = append([]string{"-n" + detachedNetNS, "--", oldCmd}, oldArgs...)
+		}
+	}
+
 	args = append([]string{cmd}, append(args, "internal", "oci-hook")...)
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
 		if s.Hooks == nil {

pkg/containerutil/container_network_manager.go

@@ -38,6 +38,7 @@ import (
 	"github.com/containerd/nerdctl/pkg/mountutil"
 	"github.com/containerd/nerdctl/pkg/netutil"
 	"github.com/containerd/nerdctl/pkg/netutil/nettype"
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -461,9 +462,37 @@ func (m *hostNetworkManager) ContainerNetworkingOpts(_ context.Context, containe
 		}
 	}
 
+	if rootlessutil.IsRootless() {
+		detachedNetNS, err := rootlessutil.DetachedNetNS()
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
+		}
+		if detachedNetNS != "" {
+			// For rootless + host netns, we can't mount /sys.
+			//
+			// TODO: consider to just bind-mount /sys from the host with rro,
+			// when rro is available (kernel >= 5.12, runc >= 1.1)
+			//
+			// Relevant: https://github.com/moby/buildkit/blob/v0.12.4/util/rootless/specconv/specconv_linux.go#L15-L34
+			specs = append(specs, withRemoveSysfs)
+		}
+	}
+
 	return specs, cOpts, nil
 }
 
+func withRemoveSysfs(_ context.Context, _ oci.Client, c *containers.Container, s *oci.Spec) error {
+	var mounts []specs.Mount // nolint: prealloc
+	for _, mount := range s.Mounts {
+		if strings.HasPrefix(mount.Destination, "/sys") {
+			continue
+		}
+		mounts = append(mounts, mount)
+	}
+	s.Mounts = mounts
+	return nil
+}
+
 // types.NetworkOptionsManager implementation for CNI networking settings.
 // This is a more specialized and OS-dependendant networking model so this
 // struct provides different implementations on different platforms.

pkg/containerutil/container_network_manager_linux.go

@@ -19,10 +19,12 @@ package containerutil
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io/fs"
 	"path/filepath"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/log"
 	"github.com/containerd/nerdctl/pkg/api/types"
@@ -32,6 +34,7 @@ import (
 	"github.com/containerd/nerdctl/pkg/netutil"
 	"github.com/containerd/nerdctl/pkg/resolvconf"
 	"github.com/containerd/nerdctl/pkg/rootlessutil"
+	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
 type cniNetworkManagerPlatform struct {
@@ -172,3 +175,23 @@ func (m *cniNetworkManager) buildResolvConf(resolvConfPath string) error {
 	_, err = resolvconf.Build(resolvConfPath, append(slirp4Dns, nameServers...), searchDomains, dnsOptions)
 	return err
 }
+
+// withRootlessKitDetachedNetNS is used when all the following conditions are satisfied:
+//   - Rootless mode
+//   - RootlessKit is running with detach-netns mode (available since RootlessKit v2.0)
+//   - the network manager is not configured to none/host/container mode
+func withRootlessKitDetachedNetNS(detachedNetNS, containerStateDir string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error {
+	return func(_ context.Context, _ oci.Client, c *containers.Container, s *oci.Spec) error {
+		for i := range s.Linux.Namespaces {
+			n := &s.Linux.Namespaces[i]
+			if n.Type == specs.NetworkNamespace && n.Path == "" {
+				newChildNetNS := filepath.Join(containerStateDir, "netns")
+				if err := netutil.NewNestedNetNS(detachedNetNS, newChildNetNS); err != nil {
+					return fmt.Errorf("failed to nest a new netns %q inside %q: %w", newChildNetNS, detachedNetNS, err)
+				}
+				n.Path = newChildNetNS
+			}
+		}
+		return nil
+	}
+}

pkg/rootlessutil/parent_linux.go

@@ -77,6 +77,16 @@ func ParentMain(hostGatewayIP string) error {
 		return err
 	}
 
+	detachedNetNSPath, err := detachedNetNS(stateDir)
+	if err != nil {
+		return err
+	}
+	detachNetNSMode := detachedNetNSPath != ""
+	log.L.Debugf("RootlessKit detach-netns mode: %v", detachNetNSMode)
+	if err != nil {
+		return err
+	}
+
 	wd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -92,10 +102,13 @@ func ParentMain(hostGatewayIP string) error {
 		"-r/",     // root dir (busybox nsenter wants this to be explicitly specified),
 		"-w" + wd, // work dir
 		"--preserve-credentials",
-		"-m", "-n", "-U",
+		"-m", "-U",
 		"-t", strconv.Itoa(childPid),
 		"-F", // no fork
 	}
+	if !detachNetNSMode {
+		args = append(args, "-n")
+	}
 	args = append(args, os.Args...)
 	log.L.Debugf("rootless parent main: executing %q with %v", arg0, args)
 

pkg/rootlessutil/rootlessutil_linux.go

@@ -17,6 +17,7 @@
 package rootlessutil
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -80,3 +81,27 @@ func RootlessContainredSockAddress() (string, error) {
 	}
 	return filepath.Join(fmt.Sprintf("/proc/%d/root/run/containerd/containerd.sock", childPid)), nil
 }
+
+// DetachedNetNS returns non-empty netns path if RootlessKit is running with --detach-netns mode.
+// Otherwise returns "" without an error.
+func DetachedNetNS() (string, error) {
+	if !IsRootless() {
+		return "", nil
+	}
+	stateDir, err := RootlessKitStateDir()
+	if err != nil {
+		return "", err
+	}
+	return detachedNetNS(stateDir)
+}
+
+func detachedNetNS(stateDir string) (string, error) {
+	p := filepath.Join(stateDir, "netns")
+	if _, err := os.Stat(p); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return "", nil
+		}
+		return "", err
+	}
+	return p, nil
+}

pkg/rootlessutil/rootlessutil_other.go

@@ -66,3 +66,7 @@ func ParentMain(hostGatewayIP string) error {
 func RootlessContainredSockAddress() (string, error) {
 	return "", fmt.Errorf("cannot inspect RootlessKit state on non-Linux hosts")
 }
+
+func DetachedNetNS() (string, error) {
+	return "", nil
+}