faster containerd start by preloading images

hack/preload-images/generate.go

@@ -29,9 +29,13 @@ import (
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/sysinit"
+	"k8s.io/minikube/pkg/util"
 )
 
 func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error {
@@ -68,44 +72,85 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 	if err != nil {
 		return errors.Wrap(err, "kubeadm images")
 	}
-
 	if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428
 		imgs = append(imgs, kic.OverlayImage)
 	}
 
+	runner := command.NewKICRunner(profile, driver.OCIBinary)
+
+	// will need to do this to enable the container run-time service
+	sv, err := util.ParseKubernetesVersion(constants.DefaultKubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "Failed to parse kubernetes version")
+	}
+
+	co := cruntime.Config{
+		Type:              containerRuntime,
+		Runner:            runner,
+		ImageRepository:   "",
+		KubernetesVersion: sv, //  this is just to satisfy cruntime and shouldnt matter what version.
+	}
+	cr, err := cruntime.New(co)
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+	if err := cr.Enable(true); err != nil {
+		exit.WithError("enable container runtime ", err)
+	}
+
 	for _, img := range imgs {
-		cmd := exec.Command("docker", "exec", profile, "docker", "pull", img)
+		cmd := imagePullCommand(containerRuntime, img)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
 		if err := cmd.Run(); err != nil {
-			return errors.Wrapf(err, "downloading %s", img)
+			return errors.Wrapf(err, "pulling image %s", img)
 		}
 	}
 
 	// Transfer in k8s binaries
 	kcfg := config.KubernetesConfig{
 		KubernetesVersion: kubernetesVersion,
 	}
-	runner := command.NewKICRunner(profile, driver.OCIBinary)
+
 	sm := sysinit.New(runner)
 
 	if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil {
 		return errors.Wrap(err, "transferring k8s binaries")
 	}
 	// Create image tarball
-	if err := createImageTarball(tarballFilename); err != nil {
+	if err := createImageTarball(tarballFilename, containerRuntime); err != nil {
 		return errors.Wrap(err, "create tarball")
 	}
+
 	return copyTarballToHost(tarballFilename)
 }
 
-func createImageTarball(tarballFilename string) error {
+// returns the right command to pull image for a specific runtime
+func imagePullCommand(containerRuntime, img string) *exec.Cmd {
+	if containerRuntime == "docker" {
+		return exec.Command("docker", "exec", profile, "docker", "pull", img)
+	}
+
+	if containerRuntime == "containerd" {
+		return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img)
+	}
+	return nil
+}
+
+func createImageTarball(tarballFilename, containerRuntime string) error {
 	// directories to save into tarball
 	dirs := []string{
-		fmt.Sprintf("./lib/docker/%s", dockerStorageDriver),
-		"./lib/docker/image",
 		"./lib/minikube/binaries",
 	}
+
+	if containerRuntime == "docker" {
+		dirs = append(dirs, fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), "./lib/docker/image")
+	}
+
+	if containerRuntime == "containerd" {
+		dirs = append(dirs, fmt.Sprintf("./lib/containerd"))
+	}
+
 	args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename}
 	args = append(args, dirs...)
 	cmd := exec.Command("docker", args...)

hack/preload-images/preload_images.go

@@ -35,7 +35,7 @@ const (
 
 var (
 	dockerStorageDriver = "overlay2"
-	containerRuntimes   = []string{"docker"}
+	containerRuntimes   = []string{"docker", "containerd"}
 	k8sVersion          string
 	k8sVersions         []string
 )
@@ -65,10 +65,10 @@ func main() {
 		for _, cr := range containerRuntimes {
 			tf := download.TarballName(kv, cr)
 			if download.PreloadExists(kv, cr) {
-				fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kv)
+				fmt.Printf("A preloaded tarball for k8s version %s - runtime %q already exists, skipping generation.\n", kv, cr)
 				continue
 			}
-			fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kv)
+			fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr)
 			if err := generateTarball(kv, cr, tf); err != nil {
 				exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr), err)
 			}

pkg/drivers/kic/kic.go

@@ -130,7 +130,7 @@ func (d *Driver) Create() error {
 			return
 		}
 		t := time.Now()
-		glog.Infof("Starting extracting preloaded images to volume")
+		glog.Infof("Starting extracting preloaded images to volume ...")
 		// Extract preloaded images to container
 		if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil {
 			glog.Infof("Unable to extract preloaded tarball to volume: %v", err)

pkg/minikube/cruntime/containerd _test.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cruntime
+
+import (
+	"testing"
+)
+
+func TestAddRepoTagToImageName(t *testing.T) {
+	var tests = []struct {
+		imgName string
+		want    string
+	}{
+		{"kubernetesui/dashboard:v2.0.0-rc6", "docker.io/kubernetesui/dashboard:v2.0.0-rc6"},
+		{"kubernetesui/metrics-scraper:v1.0.2", "docker.io/kubernetesui/metrics-scraper:v1.0.2"},
+		{"gcr.io/k8s-minikube/storage-provisioner:v1.8.1", "gcr.io/k8s-minikube/storage-provisioner:v1.8.1"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.imgName, func(t *testing.T) {
+			got := addRepoTagToImageName(tc.imgName)
+			if got != tc.want {
+				t.Errorf("expected image name to be: %q but got %q", tc.want, got)
+			}
+		})
+	}
+}

pkg/minikube/cruntime/containerd.go

@@ -19,16 +19,20 @@ package cruntime
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"os/exec"
 	"path"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/blang/semver"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -310,5 +314,118 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error {
 	if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) {
 		return nil
 	}
-	return fmt.Errorf("not yet implemented for %s", r.Name())
+
+	k8sVersion := cfg.KubernetesVersion
+	cRuntime := cfg.ContainerRuntime
+
+	// If images already exist, return
+	images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion)
+	if err != nil {
+		return errors.Wrap(err, "getting images")
+	}
+	if containerdImagesPreloaded(r.Runner, images) {
+		glog.Info("Images already preloaded, skipping extraction")
+		return nil
+	}
+
+	tarballPath := download.TarballPath(k8sVersion, cRuntime)
+	targetDir := "/"
+	targetName := "preloaded.tar.lz4"
+	dest := path.Join(targetDir, targetName)
+
+	c := exec.Command("which", "lz4")
+	if _, err := r.Runner.RunCmd(c); err != nil {
+		return NewErrISOFeature("lz4")
+	}
+
+	// Copy over tarball into host
+	fa, err := assets.NewFileAsset(tarballPath, targetDir, targetName, "0644")
+	if err != nil {
+		return errors.Wrap(err, "getting file asset")
+	}
+	t := time.Now()
+	if err := r.Runner.Copy(fa); err != nil {
+		return errors.Wrap(err, "copying file")
+	}
+	glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds())
+
+	t = time.Now()
+	// extract the tarball to /var in the VM
+	if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil {
+		return errors.Wrapf(err, "extracting tarball: %s", rr.Output())
+	}
+	glog.Infof("Took %f seconds t extract the tarball", time.Since(t).Seconds())
+
+	//  remove the tarball in the VM
+	if err := r.Runner.Remove(fa); err != nil {
+		glog.Infof("error removing tarball: %v", err)
+	}
+
+	return r.Restart()
+}
+
+// Restart restarts Docker on a host
+func (r *Containerd) Restart() error {
+	return r.Init.Restart("containerd")
+}
+
+// containerdImagesPreloaded returns true if all images have been preloaded
+func containerdImagesPreloaded(runner command.Runner, images []string) bool {
+	rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json"))
+	if err != nil {
+		return false
+	}
+	type containerdImages struct {
+		Images []struct {
+			ID          string      `json:"id"`
+			RepoTags    []string    `json:"repoTags"`
+			RepoDigests []string    `json:"repoDigests"`
+			Size        string      `json:"size"`
+			UID         interface{} `json:"uid"`
+			Username    string      `json:"username"`
+		} `json:"images"`
+	}
+
+	var jsonImages containerdImages
+	err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages)
+	if err != nil {
+		glog.Errorf("failed to unmarshal images, will assume images are not preloaded")
+		return false
+	}
+
+	// Make sure images == imgs
+	for _, i := range images {
+		found := false
+		for _, ji := range jsonImages.Images {
+			for _, rt := range ji.RepoTags {
+				i = addRepoTagToImageName(i)
+				if i == rt {
+					found = true
+					break
+				}
+			}
+			if found {
+				break
+			}
+
+		}
+		if !found {
+			glog.Infof("couldn't find preloaded image for %q. assuming images are not preloaded.", i)
+			return false
+		}
+	}
+	glog.Infof("all images are preloaded for containerd runtime.")
+	return true
+}
+
+// addRepoTagToImageName makes sure the image name has a repo tag in it.
+// in crictl images list have the repo tag prepended to them
+// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0"
+// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them
+// not mean to be used for public images
+func addRepoTagToImageName(imgName string) string {
+	if !strings.Contains(imgName, ".io/") {
+		return "docker.io/" + imgName
+	} // else it already has repo name dont add anything
+	return imgName
 }

pkg/minikube/cruntime/cruntime.go

@@ -220,3 +220,14 @@ func enableIPForwarding(cr CommandRunner) error {
 	}
 	return nil
 }
+
+// ImagesPreloaded returns true if all images have been preloaded
+func ImagesPreloaded(containerRuntime string, runner command.Runner, images []string) bool {
+	if containerRuntime == "docker" {
+		return dockerImagesPreloaded(runner, images)
+	}
+	if containerRuntime == "containerd" {
+		return containerdImagesPreloaded(runner, images)
+	}
+	return false
+}

pkg/minikube/cruntime/docker.go

@@ -290,7 +290,7 @@ func (r *Docker) Preload(cfg config.KubernetesConfig) error {
 	if err != nil {
 		return errors.Wrap(err, "getting images")
 	}
-	if DockerImagesPreloaded(r.Runner, images) {
+	if dockerImagesPreloaded(r.Runner, images) {
 		glog.Info("Images already preloaded, skipping extraction")
 		return nil
 	}
@@ -342,8 +342,8 @@ func (r *Docker) Preload(cfg config.KubernetesConfig) error {
 	return r.Restart()
 }
 
-// DockerImagesPreloaded returns true if all images have been preloaded
-func DockerImagesPreloaded(runner command.Runner, images []string) bool {
+// dockerImagesPreloaded returns true if all images have been preloaded
+func dockerImagesPreloaded(runner command.Runner, images []string) bool {
 	rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}"))
 	if err != nil {
 		return false

pkg/minikube/download/preload.go

@@ -78,16 +78,16 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string {
 
 // PreloadExists returns true if there is a preloaded tarball that can be used
 func PreloadExists(k8sVersion, containerRuntime string) bool {
+	// TODO: debug why this func is being called two times
 	glog.Infof("Checking if preload exists for k8s version %s and runtime %s", k8sVersion, containerRuntime)
 	if !viper.GetBool("preload") {
 		return false
 	}
 
-	// See https://github.com/kubernetes/minikube/issues/6933
 	// and https://github.com/kubernetes/minikube/issues/6934
-	// to track status of adding containerd & crio
-	if containerRuntime != "docker" {
-		glog.Info("Container runtime isn't docker, skipping preload")
+	// to track status of adding crio
+	if containerRuntime == "crio" {
+		glog.Info("crio is not supported yet, skipping preload")
 		return false
 	}
 

pkg/minikube/machine/cache_images.go

@@ -65,11 +65,10 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB
 // LoadImages loads previously cached images into the container runtime
 func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error {
 	// Skip loading images if images already exist
-	if cruntime.DockerImagesPreloaded(runner, images) {
+	if cruntime.ImagesPreloaded(cc.KubernetesConfig.ContainerRuntime, runner, images) {
 		glog.Infof("Images are preloaded, skipping loading")
 		return nil
 	}
-
 	glog.Infof("LoadImages start: %s", images)
 	start := time.Now()
 

test/integration/aaa_download_only_test.go

@@ -154,6 +154,7 @@ func TestDownloadOnlyKic(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
 	defer Cleanup(t, profile, cancel)
 
+	// TODO: #7795 add containerd to download only too
 	cRuntime := "docker"
 
 	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"}