Add user namespaces tests

Adding user namespaces tests for covering the `UsernamespaceMode`
supported by the CRI.

Fixes #1348

Signed-off-by: Sascha Grunert <sgrunert@redhat.com>

.github/workflows/build.yml

@@ -5,7 +5,7 @@ on:
       - "*"
     branches:
       - master
-  pull_request:
+  #pull_request:
 jobs:
   images:
     runs-on: ubuntu-latest

.github/workflows/containerd.yml

@@ -12,6 +12,7 @@ jobs:
   #
   build-and-critest-containerd:
     strategy:
+      fail-fast: false
       matrix:
         # ╔══════════════════╤═══════════╤═════════╗
         # ║ master / release │ ubuntu    │ windows ║
@@ -22,12 +23,23 @@ jobs:
         # ╟──────────────────┼───────────┼─────────╢
         # ║ hcshim           │           │ runhcs  ║
         # ╚══════════════════╧═══════════╧═════════╝
-        os: [ubuntu-22.04, windows-2019]
+        os:
+          - ubuntu-22.04
+          # - windows-2019
         # not every command likes the slash in branch name.
         # So will use format command to replace to either `_` or '/'
-        version: [main, 'release{0}1.6', 'release{0}1.7']
-        runtime: [io.containerd.runtime.v1.linux, io.containerd.runc.v1, io.containerd.runc.v2, containerd-shim-runhcs-v1]
-        runc: [runc, crun]
+        version:
+          - main
+          #- 'release{0}1.6'
+          #- 'release{0}1.7'
+        runtime:
+          - io.containerd.runc.v2
+          #- io.containerd.runtime.v1.linux
+          #- io.containerd.runc.v1
+          # - containerd-shim-runhcs-v1
+        runc:
+          - runc
+          #- crun
         exclude:
           - runtime: io.containerd.runc.v1
             os: windows-2019
@@ -298,7 +310,7 @@ jobs:
           set -o nounset
           set -o pipefail
 
-          BDIR="$(mktemp -d -p $PWD)"
+          BDIR="$(mktemp -d)"
           echo "containerd temp dir: ${BDIR}"
           mkdir -p ${BDIR}/{root,state}
 
@@ -312,16 +324,24 @@ jobs:
           # Remove possibly existing containerd configuration
           sudo rm -rf /etc/containerd
 
+          # UserNamespaces only work for 1.7 and main
+          SKIP=
+          if [[ "${{ format(matrix.version, '/') }}" == "release/1.6" ]]; then
+            SKIP=--ginkgo.skip=UserNamespaces
+          fi
+
           sudo PATH=$PATH /usr/local/bin/containerd -a ${BDIR}/c.sock -root ${BDIR}/root -state ${BDIR}/state -log-level debug &> ${BDIR}/containerd-cri.log &
           sudo /usr/local/bin/ctr -a ${BDIR}/c.sock version
           sudo /usr/local/sbin/runc --version
 
-          sudo -E PATH=$PATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
-          TEST_RC=$?
+          TEST_RC=0
+          sudo -E PATH=$PATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 ${SKIP} --ginkgo.focus=NamespaceMode_POD || TEST_RC=$?
           test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log
+          test $TEST_RC -ne 0 && sudo -E PATH=$PATH go run traverse.go $(sudo find $BDIR -type d -name rootfs)
           sudo pkill containerd
           echo "CONTD_CRI_DIR=$BDIR" >> $GITHUB_ENV
           test $TEST_RC -eq 0 || /bin/false
+        working-directory: ${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools
 
       - name: Run critest on Windows
         if: startsWith(matrix.os, 'windows')

.github/workflows/crio.yml

@@ -5,7 +5,7 @@ on:
       - "*"
     branches:
       - master
-  pull_request:
+  #pull_request:
 jobs:
   #
   # Run CRI tests against CRI-O

.github/workflows/e2e.yml

@@ -5,7 +5,7 @@ on:
       - "*"
     branches:
       - master
-  pull_request:
+  #pull_request:
 jobs:
   test-e2e:
     name: ${{ matrix.os }}

pkg/framework/util.go

@@ -220,6 +220,13 @@ func RunPodSandbox(c internalapi.RuntimeService, config *runtimeapi.PodSandboxCo
 	return podID
 }
 
+// RunPodSandboxError runs a PodSandbox and expects an error.
+func RunPodSandboxError(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string {
+	podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler)
+	Expect(err).To(HaveOccurred())
+	return podID
+}
+
 // CreatePodSandboxForContainer creates a PodSandbox for creating containers.
 func CreatePodSandboxForContainer(c internalapi.RuntimeService) (string, *runtimeapi.PodSandboxConfig) {
 	podSandboxName := "create-PodSandbox-for-container-" + NewUUID()

pkg/validate/container.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -629,6 +630,21 @@ func verifyLogContents(podConfig *runtimeapi.PodSandboxConfig, logPath string, l
 	Expect(found).To(BeTrue(), "expected log %q (stream=%q) not found in logs %+v", log, stream, msgs)
 }
 
+// verifyLogContentsRe verifies the contents of container log using the provided regular expression pattern.
+func verifyLogContentsRe(podConfig *runtimeapi.PodSandboxConfig, logPath string, pattern string, stream streamType) {
+	By("verify log contents using regex pattern")
+	msgs := parseLogLine(podConfig, logPath)
+
+	found := false
+	for _, msg := range msgs {
+		if matched, _ := regexp.MatchString(pattern, msg.log); matched && msg.stream == stream {
+			found = true
+			break
+		}
+	}
+	Expect(found).To(BeTrue(), "expected log pattern %q (stream=%q) to match logs %+v", pattern, stream, msgs)
+}
+
 // listContainerStatsForID lists container for containerID.
 func listContainerStatsForID(c internalapi.RuntimeService, containerID string) *runtimeapi.ContainerStats {
 	By("List container stats for containerID: " + containerID)

pkg/validate/security_context_linux.go

@@ -842,6 +842,100 @@ var _ = framework.KubeDescribe("Security Context", func() {
 			matchContainerOutput(podConfig, containerName, "Effective uid: 0\n")
 		})
 	})
+
+	Context("UserNamespaces", func() {
+		var (
+			podName        string
+			defaultMapping = []*runtimeapi.IDMapping{{
+				ContainerId: 0,
+				HostId:      1000,
+				Length:      100000,
+			}}
+		)
+
+		BeforeEach(func() {
+			podName = "user-namespaces-pod-" + framework.NewUUID()
+		})
+
+		It("runtime should support NamespaceMode_POD", func() {
+			namespaceOption := &runtimeapi.NamespaceOption{
+				UsernsOptions: &runtimeapi.UserNamespace{
+					Mode: runtimeapi.NamespaceMode_POD,
+					Uids: defaultMapping,
+					Gids: defaultMapping,
+				},
+			}
+
+			hostLogPath, podLogPath := createLogTempDir(podName)
+			defer os.RemoveAll(hostLogPath)
+			podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath)
+			containerName := runUserNamespaceContainer(rc, ic, podID, podConfig)
+
+			matchContainerOutputRe(podConfig, containerName, `\s+0\s+1000\s+100000\n`)
+		})
+
+		It("runtime should support NamespaceMode_NODE", func() {
+			namespaceOption := &runtimeapi.NamespaceOption{
+				UsernsOptions: &runtimeapi.UserNamespace{
+					Mode: runtimeapi.NamespaceMode_NODE,
+				},
+			}
+
+			hostLogPath, podLogPath := createLogTempDir(podName)
+			defer os.RemoveAll(hostLogPath)
+			podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath)
+			containerName := runUserNamespaceContainer(rc, ic, podID, podConfig)
+
+			// 4294967295 means that the entire range is available
+			matchContainerOutputRe(podConfig, containerName, `\s+0\s+0\s+4294967295\n`)
+		})
+
+		It("runtime should fail if more than one mapping provided", func() {
+			wrongMapping := []*runtimeapi.IDMapping{{
+				ContainerId: 0,
+				HostId:      1000,
+				Length:      100000,
+			}, {
+				ContainerId: 0,
+				HostId:      2000,
+				Length:      100000,
+			}}
+			usernsOptions := &runtimeapi.UserNamespace{
+				Mode: runtimeapi.NamespaceMode_POD,
+				Uids: wrongMapping,
+				Gids: wrongMapping,
+			}
+
+			runUserNamespacePodWithError(rc, podName, usernsOptions)
+		})
+
+		It("runtime should fail if container ID 0 is not mapped", func() {
+			mapping := []*runtimeapi.IDMapping{{
+				ContainerId: 1,
+				HostId:      1000,
+				Length:      100000,
+			}}
+			usernsOptions := &runtimeapi.UserNamespace{
+				Mode: runtimeapi.NamespaceMode_POD,
+				Uids: mapping,
+				Gids: mapping,
+			}
+
+			runUserNamespacePodWithError(rc, podName, usernsOptions)
+		})
+
+		It("runtime should fail with NamespaceMode_CONTAINER", func() {
+			usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_CONTAINER}
+
+			runUserNamespacePodWithError(rc, podName, usernsOptions)
+		})
+
+		It("runtime should fail with NamespaceMode_TARGET", func() {
+			usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_TARGET}
+
+			runUserNamespacePodWithError(rc, podName, usernsOptions)
+		})
+	})
 })
 
 // matchContainerOutput matches log line in container logs.
@@ -850,6 +944,12 @@ func matchContainerOutput(podConfig *runtimeapi.PodSandboxConfig, name, output s
 	verifyLogContents(podConfig, fmt.Sprintf("%s.log", name), output, stdoutType)
 }
 
+// matchContainerOutputRe matches log line in container logs using the provided regular expression pattern.
+func matchContainerOutputRe(podConfig *runtimeapi.PodSandboxConfig, name, pattern string) {
+	By("check container output")
+	verifyLogContentsRe(podConfig, fmt.Sprintf("%s.log", name), pattern, stdoutType)
+}
+
 // createRunAsUserContainer creates the container with specified RunAsUser in ContainerConfig.
 func createRunAsUserContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) (string, string) {
 	By("create RunAsUser container")
@@ -946,7 +1046,8 @@ func createNamespacePodSandbox(rc internalapi.RuntimeService, podSandboxNamespac
 	uid := framework.DefaultUIDPrefix + framework.NewUUID()
 	namespace := framework.DefaultNamespacePrefix + framework.NewUUID()
 	config := &runtimeapi.PodSandboxConfig{
-		Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt),
+		Metadata:  framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt),
+		DnsConfig: &runtimeapi.DNSConfig{},
 		Linux: &runtimeapi.LinuxPodSandboxConfig{
 			SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
 				NamespaceOptions: podSandboxNamespace,
@@ -1281,3 +1382,58 @@ func checkSetHostname(rc internalapi.RuntimeService, containerID string, setable
 		Expect(err).To(HaveOccurred(), msg)
 	}
 }
+
+func runUserNamespaceContainer(
+	rc internalapi.RuntimeService,
+	ic internalapi.ImageManagerService,
+	podID string,
+	podConfig *runtimeapi.PodSandboxConfig,
+) string {
+	By("create user namespaces container")
+	containerName := "user-namespaces-container-" + framework.NewUUID()
+	containerConfig := &runtimeapi.ContainerConfig{
+		Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt),
+		Image: &runtimeapi.ImageSpec{
+			Image:              framework.TestContext.TestImageList.DefaultTestContainerImage,
+			UserSpecifiedImage: framework.TestContext.TestImageList.DefaultTestContainerImage,
+		},
+		Command: []string{"cat", "/proc/self/uid_map"},
+		LogPath: fmt.Sprintf("%s.log", containerName),
+		Linux: &runtimeapi.LinuxContainerConfig{
+			SecurityContext: &runtimeapi.LinuxContainerSecurityContext{
+				NamespaceOptions: podConfig.Linux.SecurityContext.NamespaceOptions,
+			},
+		},
+	}
+
+	containerID := createContainerWithExpectation(rc, ic, containerConfig, podID, podConfig, true)
+	startContainer(rc, containerID)
+
+	Eventually(func() runtimeapi.ContainerState {
+		return getContainerStatus(rc, containerID).State
+	}, time.Minute, time.Second*4).Should(Equal(runtimeapi.ContainerState_CONTAINER_EXITED))
+
+	return containerName
+}
+
+func runUserNamespacePodWithError(
+	rc internalapi.RuntimeService,
+	podName string,
+	usernsOptions *runtimeapi.UserNamespace,
+) {
+	uid := framework.DefaultUIDPrefix + framework.NewUUID()
+	namespace := framework.DefaultNamespacePrefix + framework.NewUUID()
+	config := &runtimeapi.PodSandboxConfig{
+		Metadata: framework.BuildPodSandboxMetadata(podName, uid, namespace, framework.DefaultAttempt),
+		Linux: &runtimeapi.LinuxPodSandboxConfig{
+			SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
+				NamespaceOptions: &runtimeapi.NamespaceOption{
+					UsernsOptions: usernsOptions,
+				},
+			},
+		},
+		Labels: framework.DefaultPodLabels,
+	}
+
+	framework.RunPodSandboxError(rc, config)
+}

traverse.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func main() {
+	path := os.Args[1]
+	fmt.Printf("Using path: %v\n", path)
+	if err := traversePath(path); err != nil {
+		panic(err)
+	}
+}
+
+func traversePath(tPath string) error {
+	tempBase := os.TempDir()
+	if !strings.HasPrefix(tPath, tempBase) {
+		return fmt.Errorf("traversePath: %q is not a descendant of %q", tPath, tempBase)
+	}
+
+	var path string
+	for _, p := range strings.SplitAfter(tPath, "/") {
+		path = path + p
+		stats, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
+
+		perm := stats.Mode().Perm()
+		if perm&0o5 == 0o5 {
+			continue
+		}
+		if strings.HasPrefix(tempBase, path) {
+			return fmt.Errorf("traversePath: directory %q MUST have read+exec permissions for others", path)
+		}
+
+		if err := os.Chmod(path, perm|0o755); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}