From 3823304d216dfc78097071901818b92dbb7c8302 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sat, 18 Apr 2020 06:31:15 -0700 Subject: [PATCH 01/12] generate preload for containerd --- hack/preload-images/generate.go | 51 +++++++++++++++++++++++---- hack/preload-images/preload_images.go | 6 ++-- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 1a22e7404de4..03bf5b186a09 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -29,9 +29,13 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/sysinit" + "k8s.io/minikube/pkg/util" ) func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error { @@ -68,17 +72,42 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string if err != nil { return errors.Wrap(err, "kubeadm images") } - if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428 imgs = append(imgs, kic.OverlayImage) } + runner := command.NewKICRunner(profile, driver.OCIBinary) + + // will need to do this to enable the container run-time service + sv, err := util.ParseKubernetesVersion(constants.DefaultKubernetesVersion) + if err != nil { + return errors.Wrap(err, "Failed to parse kubernetes version") + } + + co := cruntime.Config{ + Type: containerRuntime, + Runner: runner, + ImageRepository: "", + KubernetesVersion: sv, // I think this is just to statsify cruntime and shouldnt matter + } + cr, err := cruntime.New(co) + if err != nil { + exit.WithError("Failed runtime", err) + } + if err := cr.Enable(true); err != nil { + exit.WithError("enable container runtime ", err) + } + for _, img := range imgs { cmd := exec.Command("docker", "exec", profile, "docker", "pull", img) + if containerRuntime == "containerd" { + cmd = exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) + } + cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "downloading %s", img) + return errors.Wrapf(err, "pulling image %s", img) } } @@ -86,26 +115,34 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string kcfg := config.KubernetesConfig{ KubernetesVersion: kubernetesVersion, } - runner := command.NewKICRunner(profile, driver.OCIBinary) + sm := sysinit.New(runner) if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil { return errors.Wrap(err, "transferring k8s binaries") } // Create image tarball - if err := createImageTarball(tarballFilename); err != nil { + if err := createImageTarball(tarballFilename, containerRuntime); err != nil { return errors.Wrap(err, "create tarball") } + return copyTarballToHost(tarballFilename) } -func createImageTarball(tarballFilename string) error { +func createImageTarball(tarballFilename, containerRuntime string) error { // directories to save into tarball dirs := []string{ - fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), - "./lib/docker/image", "./lib/minikube/binaries", } + + if containerRuntime == "docker" { + dirs = append(dirs, fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), "./lib/docker/image") + } + + if containerRuntime == "containerd" { + dirs = append(dirs, fmt.Sprintf("./lib/containerd")) + } + args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} args = append(args, dirs...) cmd := exec.Command("docker", args...) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 37bbd678e7d6..cc688174b61a 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -35,7 +35,7 @@ const ( var ( dockerStorageDriver = "overlay2" - containerRuntimes = []string{"docker"} + containerRuntimes = []string{"containerd"} k8sVersion string k8sVersions []string ) @@ -65,10 +65,10 @@ func main() { for _, cr := range containerRuntimes { tf := download.TarballName(kv, cr) if download.PreloadExists(kv, cr) { - fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kv) + fmt.Printf("A preloaded tarball for k8s version %s - runtime %q already exists, skipping generation.\n", kv, cr) continue } - fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kv) + fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr) if err := generateTarball(kv, cr, tf); err != nil { exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr), err) } From c017c98e81bbbcb9504b16ffd33da7864cfaa50c Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sun, 19 Apr 2020 19:33:11 -0700 Subject: [PATCH 02/12] add preload funcs for containerd --- pkg/drivers/kic/oci/volumes.go | 2 + pkg/minikube/cruntime/containerd.go | 80 ++++++++++++++++++++++++++++- pkg/minikube/cruntime/docker.go | 1 + pkg/minikube/download/preload.go | 7 ++- 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 2fbd9f32b057..bb6627d4601e 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -81,8 +81,10 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { // ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath // to the volume named volumeName func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { + fmt.Printf("\n-------------------\n(medya dbg) inside ExtractTarballToVolume. tarballPath: %s voluneName %s \n-------------------\n", tarballPath, volumeName) cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") glog.Infof("executing: %s", cmd.Args) + fmt.Printf("medya dbg \n---------%s ----------\n", cmd.Args) if out, err := cmd.CombinedOutput(); err != nil { return errors.Wrapf(err, "output %s", string(out)) } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 8eadb57392e2..5d5064db8f5f 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -24,11 +24,14 @@ import ( "path" "strings" "text/template" + "time" "github.com/blang/semver" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" @@ -310,5 +313,80 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error { if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } - return fmt.Errorf("not yet implemented for %s", r.Name()) + fmt.Println("(medya dbg) inside Preload docker") + if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { + return nil + } + k8sVersion := cfg.KubernetesVersion + cRuntime := cfg.ContainerRuntime + + // If images already exist, return + images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion) + if err != nil { + return errors.Wrap(err, "getting images") + } + if ContainerdImagesPreloaded(r.Runner, images) { + glog.Info("Images already preloaded, skipping extraction") + return nil + } + + tarballPath := download.TarballPath(k8sVersion, cRuntime) + targetDir := "/" + targetName := "preloaded.tar.lz4" + dest := path.Join(targetDir, targetName) + + c := exec.Command("which", "lz4") + if _, err := r.Runner.RunCmd(c); err != nil { + return NewErrISOFeature("lz4") + } + + // Copy over tarball into host + fa, err := assets.NewFileAsset(tarballPath, targetDir, targetName, "0644") + if err != nil { + return errors.Wrap(err, "getting file asset") + } + t := time.Now() + if err := r.Runner.Copy(fa); err != nil { + return errors.Wrap(err, "copying file") + } + glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds()) + + t = time.Now() + // extract the tarball to /var in the VM + if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil { + return errors.Wrapf(err, "extracting tarball: %s", rr.Output()) + } + glog.Infof("Took %f seconds t extract the tarball", time.Since(t).Seconds()) + + // remove the tarball in the VM + if err := r.Runner.Remove(fa); err != nil { + glog.Infof("error removing tarball: %v", err) + } + + return r.Restart() +} + +// Restart restarts Docker on a host +func (r *Containerd) Restart() error { + return r.Init.Restart("containerd") +} + +// ContainerdImagesPreloaded returns true if all images have been preloaded +func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { + fmt.Printf("medyadb inside ContainerdImagesPreloaded : images %s \n", strings.Join(images, ",")) + // rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images")) + // if err != nil { + // return false + // } + + // glog.Infof("Got contained preloaded images: %s", rr.Output()) + + // // Make sure images == imgs + // for _, i := range images { + // if !strings.Contains(rr.Output(), i) { + // glog.Infof("%s wasn't preloaded", i) + // return false + // } + // } + return true } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 95f855415235..080cae855685 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -279,6 +279,7 @@ func (r *Docker) SystemLogCmd(len int) string { // 2. Extract the preloaded tarball to the correct directory // 3. Remove the tarball within the VM func (r *Docker) Preload(cfg config.KubernetesConfig) error { + fmt.Println("(medya dbg) inside Preload docker") if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 59a0c4b7e098..dfdcf96f95f7 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -83,11 +83,10 @@ func PreloadExists(k8sVersion, containerRuntime string) bool { return false } - // See https://github.com/kubernetes/minikube/issues/6933 // and https://github.com/kubernetes/minikube/issues/6934 - // to track status of adding containerd & crio - if containerRuntime != "docker" { - glog.Info("Container runtime isn't docker, skipping preload") + // to track status of adding crio + if containerRuntime == "crio" { + glog.Info("crio is not supported yet, skipping preload") return false } From ac66bf2f906b9ae9d00f77e4493671dd1184fd82 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sun, 19 Apr 2020 19:59:50 -0700 Subject: [PATCH 03/12] first working version --- pkg/drivers/kic/kic.go | 2 +- pkg/minikube/cruntime/containerd.go | 16 ++++++++++------ pkg/minikube/download/preload.go | 5 +++++ pkg/minikube/machine/cache_images.go | 18 +++++++++++++----- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 05b1e325ca28..d14f0e4ce48a 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -130,7 +130,7 @@ func (d *Driver) Create() error { return } t := time.Now() - glog.Infof("Starting extracting preloaded images to volume") + glog.Infof("Starting extracting preloaded images to volume ....") // Extract preloaded images to container if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil { glog.Infof("Unable to extract preloaded tarball to volume: %v", err) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 5d5064db8f5f..b7e2f6c17419 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -22,6 +22,7 @@ import ( "fmt" "os/exec" "path" + "runtime/debug" "strings" "text/template" "time" @@ -232,6 +233,7 @@ func (r *Containerd) ImageExists(name string, sha string) bool { // LoadImage loads an image into this runtime func (r *Containerd) LoadImage(path string) error { + debug.PrintStack() glog.Infof("Loading image: %s", path) c := exec.Command("sudo", "ctr", "-n=k8s.io", "images", "import", path) if _, err := r.Runner.RunCmd(c); err != nil { @@ -327,7 +329,7 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error { } if ContainerdImagesPreloaded(r.Runner, images) { glog.Info("Images already preloaded, skipping extraction") - return nil + return r.Restart() } tarballPath := download.TarballPath(k8sVersion, cRuntime) @@ -374,12 +376,14 @@ func (r *Containerd) Restart() error { // ContainerdImagesPreloaded returns true if all images have been preloaded func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { fmt.Printf("medyadb inside ContainerdImagesPreloaded : images %s \n", strings.Join(images, ",")) - // rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images")) - // if err != nil { - // return false - // } + fmt.Println("=================================") + rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images")) + if err != nil { + return false + } - // glog.Infof("Got contained preloaded images: %s", rr.Output()) + glog.Infof("Got contained preloaded images: %s", rr.Output()) + fmt.Println("=================================") // // Make sure images == imgs // for _, i := range images { diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index dfdcf96f95f7..23ca0dbc35fa 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "runtime" + "runtime/debug" "cloud.google.com/go/storage" "google.golang.org/api/option" @@ -78,6 +79,10 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string { // PreloadExists returns true if there is a preloaded tarball that can be used func PreloadExists(k8sVersion, containerRuntime string) bool { + // TODO: debug why this func is being called two times + fmt.Println("************************************************************************") + debug.PrintStack() + fmt.Println("************************************************************************") glog.Infof("Checking if preload exists for k8s version %s and runtime %s", k8sVersion, containerRuntime) if !viper.GetBool("preload") { return false diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 93f735467c7b..5a403b886589 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -64,12 +64,20 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { - // Skip loading images if images already exist - if cruntime.DockerImagesPreloaded(runner, images) { - glog.Infof("Images are preloaded, skipping loading") - return nil + if cc.KubernetesConfig.ContainerRuntime == "docker" { + // Skip loading images if images already exist + if cruntime.DockerImagesPreloaded(runner, images) { + glog.Infof("Images are preloaded, skipping loading") + return nil + } + } + if cc.KubernetesConfig.ContainerRuntime == "containerd" { + // Skip loading images if images already exist + if cruntime.ContainerdImagesPreloaded(runner, images) { + glog.Infof("Images are preloaded, skipping loading") + return nil + } } - glog.Infof("LoadImages start: %s", images) start := time.Now() From d2415d778e3d993b2cdd7ae61c9db40e36d1a7c7 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sun, 19 Apr 2020 20:36:22 -0700 Subject: [PATCH 04/12] add json parser for critcl images --- pkg/drivers/kic/oci/volumes.go | 2 - pkg/minikube/cruntime/containerd.go | 64 ++++++++++++++++++++--------- pkg/minikube/cruntime/docker.go | 1 - pkg/minikube/download/preload.go | 4 -- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index bb6627d4601e..2fbd9f32b057 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -81,10 +81,8 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { // ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath // to the volume named volumeName func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { - fmt.Printf("\n-------------------\n(medya dbg) inside ExtractTarballToVolume. tarballPath: %s voluneName %s \n-------------------\n", tarballPath, volumeName) cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") glog.Infof("executing: %s", cmd.Args) - fmt.Printf("medya dbg \n---------%s ----------\n", cmd.Args) if out, err := cmd.CombinedOutput(); err != nil { return errors.Wrapf(err, "output %s", string(out)) } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index b7e2f6c17419..7ccb98ef087c 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -19,10 +19,10 @@ package cruntime import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "os/exec" "path" - "runtime/debug" "strings" "text/template" "time" @@ -233,7 +233,6 @@ func (r *Containerd) ImageExists(name string, sha string) bool { // LoadImage loads an image into this runtime func (r *Containerd) LoadImage(path string) error { - debug.PrintStack() glog.Infof("Loading image: %s", path) c := exec.Command("sudo", "ctr", "-n=k8s.io", "images", "import", path) if _, err := r.Runner.RunCmd(c); err != nil { @@ -315,10 +314,7 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error { if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } - fmt.Println("(medya dbg) inside Preload docker") - if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { - return nil - } + k8sVersion := cfg.KubernetesVersion cRuntime := cfg.ContainerRuntime @@ -329,7 +325,7 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error { } if ContainerdImagesPreloaded(r.Runner, images) { glog.Info("Images already preloaded, skipping extraction") - return r.Restart() + return nil } tarballPath := download.TarballPath(k8sVersion, cRuntime) @@ -375,22 +371,52 @@ func (r *Containerd) Restart() error { // ContainerdImagesPreloaded returns true if all images have been preloaded func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { - fmt.Printf("medyadb inside ContainerdImagesPreloaded : images %s \n", strings.Join(images, ",")) - fmt.Println("=================================") - rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images")) + rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json")) if err != nil { return false } + type containerdImages struct { + Images []struct { + ID string `json:"id"` + RepoTags []string `json:"repoTags"` + RepoDigests []string `json:"repoDigests"` + Size string `json:"size"` + UID interface{} `json:"uid"` + Username string `json:"username"` + } `json:"images"` + } - glog.Infof("Got contained preloaded images: %s", rr.Output()) - fmt.Println("=================================") + var jsonImages containerdImages + err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages) + if err != nil { + glog.Errorf("failed to unmarshal images, will assume images are not preloaded") + return false + } + + // Make sure images == imgs + for _, i := range images { + found := false + for _, ji := range jsonImages.Images { + for _, rt := range ji.RepoTags { + // for exmaple kubernetesui/dashboard:v2.0.0-rc6 will show up as docker.io/kubernetesui/dashboard:v2.0.0-rc6 + if !strings.Contains(i, ".io/") { + i = "docker.io/" + i + } + if i == rt { + found = true + break + } + } + if found { + break + } - // // Make sure images == imgs - // for _, i := range images { - // if !strings.Contains(rr.Output(), i) { - // glog.Infof("%s wasn't preloaded", i) - // return false - // } - // } + } + if !found { + glog.Infof("couldn't find preloaded image for %q. assuming images are not preloaded.", i) + return false + } + } + glog.Infof("all images are preloaded for containerd runtime.") return true } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 080cae855685..95f855415235 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -279,7 +279,6 @@ func (r *Docker) SystemLogCmd(len int) string { // 2. Extract the preloaded tarball to the correct directory // 3. Remove the tarball within the VM func (r *Docker) Preload(cfg config.KubernetesConfig) error { - fmt.Println("(medya dbg) inside Preload docker") if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 23ca0dbc35fa..50a12a37bb60 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -25,7 +25,6 @@ import ( "os" "path/filepath" "runtime" - "runtime/debug" "cloud.google.com/go/storage" "google.golang.org/api/option" @@ -80,9 +79,6 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string { // PreloadExists returns true if there is a preloaded tarball that can be used func PreloadExists(k8sVersion, containerRuntime string) bool { // TODO: debug why this func is being called two times - fmt.Println("************************************************************************") - debug.PrintStack() - fmt.Println("************************************************************************") glog.Infof("Checking if preload exists for k8s version %s and runtime %s", k8sVersion, containerRuntime) if !viper.GetBool("preload") { return false From bc36eb2de9634dbd3dfe252fb897b2cb5910a481 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sun, 19 Apr 2020 21:13:46 -0700 Subject: [PATCH 05/12] add todo comment --- pkg/minikube/cruntime/containerd.go | 3 ++- test/integration/aaa_download_only_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 7ccb98ef087c..c00a2537360c 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -398,7 +398,8 @@ func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { found := false for _, ji := range jsonImages.Images { for _, rt := range ji.RepoTags { - // for exmaple kubernetesui/dashboard:v2.0.0-rc6 will show up as docker.io/kubernetesui/dashboard:v2.0.0-rc6 + // in crictl docker.io is appened to images without repo organization + // for example "kubernetesui/dashboard:v2.0.0-rc6 will show up as "docker.io/kubernetesui/dashboard:v2.0.0-rc6" if !strings.Contains(i, ".io/") { i = "docker.io/" + i } diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go index 7d198b76343c..9d5ec2844373 100644 --- a/test/integration/aaa_download_only_test.go +++ b/test/integration/aaa_download_only_test.go @@ -154,6 +154,7 @@ func TestDownloadOnlyKic(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) defer Cleanup(t, profile, cancel) + // TODO: #7795 add containerd to download only too cRuntime := "docker" args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"} From 1bdd9ff7e8dba9deb1be59ae760197ca170a0e59 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 12:25:19 -0700 Subject: [PATCH 06/12] add docker back --- hack/preload-images/preload_images.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index cc688174b61a..b5fa25698709 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -35,7 +35,7 @@ const ( var ( dockerStorageDriver = "overlay2" - containerRuntimes = []string{"containerd"} + containerRuntimes = []string{"docker", "containerd"} k8sVersion string k8sVersions []string ) From 366dc8d127ecfce28f08d7885f8190a9744f68bf Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 13:13:37 -0700 Subject: [PATCH 07/12] add helper for img plull command --- hack/images/kicbase.Dockerfile | 2 +- hack/preload-images/generate.go | 20 ++++++++++++++------ pkg/drivers/kic/kic.go | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 97a29b73710b..54539660bea5 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -21,7 +21,7 @@ RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/lib apt-key add - < Release.key && apt-get update && \ apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1 # install podman -RUN apt-get install -y --no-install-recommends podman=1.8.2~144 +RUN apt-get install -y --no-install-recommends podman # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml # enable docker which is default diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 03bf5b186a09..c5ec78a95ed2 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -88,7 +88,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string Type: containerRuntime, Runner: runner, ImageRepository: "", - KubernetesVersion: sv, // I think this is just to statsify cruntime and shouldnt matter + KubernetesVersion: sv, // this is just to satsify cruntime and shouldnt matter what version. } cr, err := cruntime.New(co) if err != nil { @@ -99,11 +99,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string } for _, img := range imgs { - cmd := exec.Command("docker", "exec", profile, "docker", "pull", img) - if containerRuntime == "containerd" { - cmd = exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) - } - + cmd := imagePullCommand(containerRuntime, img) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { @@ -129,6 +125,18 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string return copyTarballToHost(tarballFilename) } +// returns the right command to pull image for a specific runtime +func imagePullCommand(containerRuntime, img string) *exec.Cmd { + if containerRuntime == "docker" { + return exec.Command("docker", "exec", profile, "docker", "pull", img) + } + + if containerRuntime == "containerd" { + return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) + } + return nil +} + func createImageTarball(tarballFilename, containerRuntime string) error { // directories to save into tarball dirs := []string{ diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index d14f0e4ce48a..0c1a4e4f487e 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -130,7 +130,7 @@ func (d *Driver) Create() error { return } t := time.Now() - glog.Infof("Starting extracting preloaded images to volume ....") + glog.Infof("Starting extracting preloaded images to volume ...") // Extract preloaded images to container if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil { glog.Infof("Unable to extract preloaded tarball to volume: %v", err) From 16c20771061450355c5f295405468e5a239a8043 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 13:32:55 -0700 Subject: [PATCH 08/12] add helpers and unit test --- pkg/minikube/cruntime/containerd _test.go | 40 +++++++++++++++++++++++ pkg/minikube/cruntime/containerd.go | 18 +++++++--- pkg/minikube/cruntime/cruntime.go | 11 +++++++ pkg/minikube/machine/cache_images.go | 17 +++------- 4 files changed, 68 insertions(+), 18 deletions(-) create mode 100644 pkg/minikube/cruntime/containerd _test.go diff --git a/pkg/minikube/cruntime/containerd _test.go b/pkg/minikube/cruntime/containerd _test.go new file mode 100644 index 000000000000..cfb1e55f6706 --- /dev/null +++ b/pkg/minikube/cruntime/containerd _test.go @@ -0,0 +1,40 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cruntime + +import ( + "testing" +) + +func TestAddRepoTagToImageName(t *testing.T) { + var tests = []struct { + imgName string + want string + }{ + {"kubernetesui/dashboard:v2.0.0-rc6", "docker.io/kubernetesui/dashboard:v2.0.0-rc6"}, + {"kubernetesui/metrics-scraper:v1.0.2", "docker.io/kubernetesui/metrics-scraper:v1.0.2"}, + {"gcr.io/k8s-minikube/storage-provisioner:v1.8.1", "gcr.io/k8s-minikube/storage-provisioner:v1.8.1"}, + } + for _, tc := range tests { + t.Run(tc.imgName, func(t *testing.T) { + got := addRepoTagToImageName(tc.imgName) + if got != tc.want { + t.Errorf("expected image name to be: %q but got %q", tc.want, got) + } + }) + } +} diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index c00a2537360c..4c14bfd5196c 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -398,11 +398,7 @@ func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { found := false for _, ji := range jsonImages.Images { for _, rt := range ji.RepoTags { - // in crictl docker.io is appened to images without repo organization - // for example "kubernetesui/dashboard:v2.0.0-rc6 will show up as "docker.io/kubernetesui/dashboard:v2.0.0-rc6" - if !strings.Contains(i, ".io/") { - i = "docker.io/" + i - } + i = addRepoTagToImageName(i) if i == rt { found = true break @@ -421,3 +417,15 @@ func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { glog.Infof("all images are preloaded for containerd runtime.") return true } + +// addRepoTagToImageName makes sure the image name has a repo tag in it. +// in crictl images list have the repo tag prepended to them +// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0" +// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them +// not mean to be used for public images +func addRepoTagToImageName(imgName string) string { + if !strings.Contains(imgName, ".io/") { + return "docker.io/" + imgName + } // else it already has repo name dont add anything + return imgName +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index bbb410be131e..6d1a3facca85 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -220,3 +220,14 @@ func enableIPForwarding(cr CommandRunner) error { } return nil } + +// ImagesPreloaded returns true if all images have been preloaded +func ImagesPreloaded(containerRuntime string, runner command.Runner, images []string) bool { + if containerRuntime == "docker" { + return DockerImagesPreloaded(runner, images) + } + if containerRuntime == "containerd" { + return ContainerdImagesPreloaded(runner, images) + } + return false +} diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 5a403b886589..94702cbc077a 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -64,19 +64,10 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { - if cc.KubernetesConfig.ContainerRuntime == "docker" { - // Skip loading images if images already exist - if cruntime.DockerImagesPreloaded(runner, images) { - glog.Infof("Images are preloaded, skipping loading") - return nil - } - } - if cc.KubernetesConfig.ContainerRuntime == "containerd" { - // Skip loading images if images already exist - if cruntime.ContainerdImagesPreloaded(runner, images) { - glog.Infof("Images are preloaded, skipping loading") - return nil - } + // Skip loading images if images already exist + if cruntime.ImagesPreloaded(cc.KubernetesConfig.ContainerRuntime, runner, images) { + glog.Infof("Images are preloaded, skipping loading") + return nil } glog.Infof("LoadImages start: %s", images) start := time.Now() From c504f2febd1e6c47c62b7d15ee654499d364579b Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 13:33:42 -0700 Subject: [PATCH 09/12] spell --- hack/preload-images/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index c5ec78a95ed2..c3e2efa71838 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -88,7 +88,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string Type: containerRuntime, Runner: runner, ImageRepository: "", - KubernetesVersion: sv, // this is just to satsify cruntime and shouldnt matter what version. + KubernetesVersion: sv, // this is just to satisfy cruntime and shouldnt matter what version. } cr, err := cruntime.New(co) if err != nil { From 22c28501023a58e343025a636a025d5108bc84a0 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 13:35:15 -0700 Subject: [PATCH 10/12] revert unrelated changes --- hack/images/kicbase.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 54539660bea5..97a29b73710b 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -21,7 +21,7 @@ RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/lib apt-key add - < Release.key && apt-get update && \ apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1 # install podman -RUN apt-get install -y --no-install-recommends podman +RUN apt-get install -y --no-install-recommends podman=1.8.2~144 # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml # enable docker which is default From ba2e27025f17c34d80faee9985e261a18fdca8fa Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 13:37:46 -0700 Subject: [PATCH 11/12] make funcs private --- pkg/minikube/cruntime/containerd.go | 6 +++--- pkg/minikube/cruntime/cruntime.go | 4 ++-- pkg/minikube/cruntime/docker.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 4c14bfd5196c..73261c98190c 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -323,7 +323,7 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "getting images") } - if ContainerdImagesPreloaded(r.Runner, images) { + if containerdImagesPreloaded(r.Runner, images) { glog.Info("Images already preloaded, skipping extraction") return nil } @@ -369,8 +369,8 @@ func (r *Containerd) Restart() error { return r.Init.Restart("containerd") } -// ContainerdImagesPreloaded returns true if all images have been preloaded -func ContainerdImagesPreloaded(runner command.Runner, images []string) bool { +// containerdImagesPreloaded returns true if all images have been preloaded +func containerdImagesPreloaded(runner command.Runner, images []string) bool { rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json")) if err != nil { return false diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 6d1a3facca85..f1866ff1aacc 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -224,10 +224,10 @@ func enableIPForwarding(cr CommandRunner) error { // ImagesPreloaded returns true if all images have been preloaded func ImagesPreloaded(containerRuntime string, runner command.Runner, images []string) bool { if containerRuntime == "docker" { - return DockerImagesPreloaded(runner, images) + return dockerImagesPreloaded(runner, images) } if containerRuntime == "containerd" { - return ContainerdImagesPreloaded(runner, images) + return containerdImagesPreloaded(runner, images) } return false } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 95f855415235..eb11b58e839f 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -290,7 +290,7 @@ func (r *Docker) Preload(cfg config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "getting images") } - if DockerImagesPreloaded(r.Runner, images) { + if dockerImagesPreloaded(r.Runner, images) { glog.Info("Images already preloaded, skipping extraction") return nil } @@ -342,8 +342,8 @@ func (r *Docker) Preload(cfg config.KubernetesConfig) error { return r.Restart() } -// DockerImagesPreloaded returns true if all images have been preloaded -func DockerImagesPreloaded(runner command.Runner, images []string) bool { +// dockerImagesPreloaded returns true if all images have been preloaded +func dockerImagesPreloaded(runner command.Runner, images []string) bool { rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}")) if err != nil { return false From 49fa90a384538e76c6ab9d3bdaf3eb4f487ae124 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Mon, 20 Apr 2020 14:57:10 -0700 Subject: [PATCH 12/12] fix file name --- pkg/minikube/cruntime/{containerd _test.go => containerd_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/minikube/cruntime/{containerd _test.go => containerd_test.go} (100%) diff --git a/pkg/minikube/cruntime/containerd _test.go b/pkg/minikube/cruntime/containerd_test.go similarity index 100% rename from pkg/minikube/cruntime/containerd _test.go rename to pkg/minikube/cruntime/containerd_test.go