gvisor: Use chroot instead of LD_LIBRARY_PATH

Makefile

@@ -62,6 +62,8 @@ GOLINT_OPTIONS = --timeout 4m \
 	  --enable goimports,gocritic,golint,gocyclo,misspell,nakedret,stylecheck,unconvert,unparam,dogsled \
 	  --exclude 'variable on range scope.*in function literal|ifElseChain'
 
+# Major version of gvisor image. Increment when there are breaking changes.
+GVISOR_IMAGE_VERSION ?= 2
 
 export GO111MODULE := on
 
@@ -480,11 +482,11 @@ out/gvisor-addon: pkg/minikube/assets/assets.go pkg/minikube/translate/translati
 
 .PHONY: gvisor-addon-image
 gvisor-addon-image: out/gvisor-addon
-	docker build -t $(REGISTRY)/gvisor-addon:latest -f deploy/gvisor/Dockerfile .
+	docker build -t $(REGISTRY)/gvisor-addon:$(GVISOR_IMAGE_VERSION) -f deploy/gvisor/Dockerfile .
 
 .PHONY: push-gvisor-addon-image
 push-gvisor-addon-image: gvisor-addon-image
-	gcloud docker -- push $(REGISTRY)/gvisor-addon:latest
+	gcloud docker -- push $(REGISTRY)/gvisor-addon:$(GVISOR_IMAGE_VERSION)
 
 .PHONY: release-iso
 release-iso: minikube_iso checksum

deploy/addons/gvisor/gvisor-pod.yaml.tmpl

@@ -24,50 +24,28 @@ spec:
   hostPID: true
   containers:
     - name: gvisor
-      image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest
+      image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:2
       securityContext:
         privileged: true
       volumeMounts:
       - mountPath: /node/
-        name: node
-      - mountPath: /usr/libexec/sudo
-        name: sudo
-      - mountPath: /var/run
-        name: varrun
-      - mountPath: /usr/bin
-        name: usrbin
-      - mountPath: /usr/lib
-        name: usrlib
-      - mountPath: /bin
-        name: bin
+        name: node-root
+      - mountPath: /node/run
+        name: node-run
       - mountPath: /tmp/gvisor
-        name: gvisor
+        name: node-tmp
       env:
-        - name: PATH
-          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin
         - name: SYSTEMD_IGNORE_CHROOT
           value: "yes"
       imagePullPolicy: IfNotPresent
   volumes:
-  - name: node
+  - name: node-root
     hostPath:
       path: /
-  - name: sudo
+  - name: node-run
     hostPath:
-      path: /usr/libexec/sudo
-  - name: varrun
-    hostPath:
-      path: /var/run
-  - name: usrlib
-    hostPath:
-      path: /usr/lib
-  - name: usrbin
-    hostPath:
-      path: /usr/bin
-  - name: bin
-    hostPath:
-      path: /bin
-  - name: gvisor
+      path: /run
+  - name: node-tmp
     hostPath:
       path: /tmp/gvisor
   restartPolicy: Always

deploy/gvisor/Dockerfile

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:18.04
-RUN apt-get update && \
-    apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \
-    rm -rf /var/lib/apt/lists/*
+# Need an image with chroot
+FROM alpine:3
 COPY out/gvisor-addon /gvisor-addon
 CMD ["/gvisor-addon"]

hack/jenkins/common.sh

@@ -245,13 +245,13 @@ mkdir -p "${TEST_HOME}"
 export MINIKUBE_HOME="${TEST_HOME}/.minikube"
 export KUBECONFIG="${TEST_HOME}/kubeconfig"
 
-# Build the gvisor image. This will be copied into minikube and loaded by ctr.
-# Used by TestContainerd for Gvisor Test.
-# TODO: move this to integration test setup.
+
+# Build the gvisor image so that we can integration test changes to pkg/gvisor
 chmod +x ./testdata/gvisor-addon
 # skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137
 if [ "$(uname)" != "Darwin" ]; then
-  docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile ./testdata
+  # Should match GVISOR_IMAGE_VERSION in Makefile
+  docker build -t gcr.io/k8s-minikube/gvisor-addon:2 -f testdata/gvisor-addon-Dockerfile ./testdata
 fi
 
 echo ""

pkg/gvisor/enable.go

@@ -157,7 +157,7 @@ func copyConfigFiles() error {
 	if err := mcnutils.CopyFile(filepath.Join(nodeDir, containerdConfigTomlPath), filepath.Join(nodeDir, storedContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying default config.toml")
 	}
-	log.Print("Copying containerd config.toml with gvisor...")
+	log.Printf("Copying %s asset to %s", constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, containerdConfigTomlPath))
 	if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, containerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying gvisor version of config.toml")
 	}
@@ -171,8 +171,13 @@ func copyAssetToDest(targetName, dest string) error {
 			asset = a
 		}
 	}
+	if asset == nil {
+		return fmt.Errorf("no asset matching target %s among %+v", targetName, assets.Addons["gvisor"])
+	}
+
 	// Now, copy the data from this asset to dest
 	src := filepath.Join(constants.GvisorFilesPath, asset.GetTargetName())
+	log.Printf("%s asset path: %s", targetName, src)
 	contents, err := ioutil.ReadFile(src)
 	if err != nil {
 		return errors.Wrapf(err, "getting contents of %s", asset.GetAssetName())
@@ -182,6 +187,8 @@ func copyAssetToDest(targetName, dest string) error {
 			return errors.Wrapf(err, "removing %s", dest)
 		}
 	}
+
+	log.Printf("creating %s", dest)
 	f, err := os.Create(dest)
 	if err != nil {
 		return errors.Wrapf(err, "creating %s", dest)
@@ -193,28 +200,24 @@ func copyAssetToDest(targetName, dest string) error {
 }
 
 func restartContainerd() error {
-	dir := filepath.Join(nodeDir, "usr/libexec/sudo")
-	if err := os.Setenv("LD_LIBRARY_PATH", dir); err != nil {
-		return errors.Wrap(err, dir)
-	}
+	log.Print("restartContainerd black magic happening")
 
 	log.Print("Stopping rpc-statd.service...")
-	// first, stop  rpc-statd.service
-	cmd := exec.Command("sudo", "-E", "systemctl", "stop", "rpc-statd.service")
+	cmd := exec.Command("/usr/sbin/chroot", "/node", "sudo", "systemctl", "stop", "rpc-statd.service")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		fmt.Println(string(out))
 		return errors.Wrap(err, "stopping rpc-statd.service")
 	}
-	// restart containerd
+
 	log.Print("Restarting containerd...")
-	cmd = exec.Command("sudo", "-E", "systemctl", "restart", "containerd")
+	cmd = exec.Command("/usr/sbin/chroot", "/node", "sudo", "systemctl", "restart", "containerd")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		log.Print(string(out))
 		return errors.Wrap(err, "restarting containerd")
 	}
-	// start rpc-statd.service
+
 	log.Print("Starting rpc-statd...")
-	cmd = exec.Command("sudo", "-E", "systemctl", "start", "rpc-statd.service")
+	cmd = exec.Command("/usr/sbin/chroot", "/node", "sudo", "systemctl", "start", "rpc-statd.service")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		log.Print(string(out))
 		return errors.Wrap(err, "restarting rpc-statd.service")

test/integration/gvisor_addon_test.go

@@ -35,6 +35,13 @@ func TestGvisorAddon(t *testing.T) {
 	profile := UniqueProfileName("gvisor")
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer func() {
+		if t.Failed() {
+			rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "gvisor", "-n", "kube-system"))
+			if err != nil {
+				t.Logf("failed to get gvisor post-mortem logs: %v", err)
+			}
+			t.Logf("gvisor post-mortem: %s:\n%s\n", rr.Command(), rr.Output())
+		}
 		CleanupWithLogs(t, profile, cancel)
 	}()