Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use preloaded tarball for VMs as well #6898

Merged
merged 13 commits into from
Mar 6, 2020
21 changes: 15 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.13

require (
cloud.google.com/go v0.45.1
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
github.com/Parallels/docker-machine-parallels v1.3.0
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/blang/semver v3.5.0+incompatible
Expand All @@ -12,6 +13,7 @@ require (
github.com/cheggaaa/pb/v3 v3.0.1
github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57 // indirect
github.com/docker/cli v0.0.0-20200303162255-7d407207c304 // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-units v0.4.0
Expand All @@ -20,9 +22,10 @@ require (
github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
github.com/evanphx/json-patch v4.5.0+incompatible // indirect
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/gogo/protobuf v1.3.1 // indirect
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/google/go-cmp v0.3.0
github.com/google/go-cmp v0.3.1
github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2
github.com/googleapis/gnostic v0.3.0 // indirect
github.com/hashicorp/go-getter v1.4.0
Expand All @@ -47,29 +50,35 @@ require (
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
github.com/onsi/ginkgo v1.10.3 // indirect
github.com/onsi/gomega v1.7.1 // indirect
github.com/otiai10/copy v1.0.2
github.com/pborman/uuid v1.2.0
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/pkg/browser v0.0.0-20160118053552-9302be274faa
github.com/pkg/errors v0.8.1
github.com/pkg/errors v0.9.1
github.com/pkg/profile v0.0.0-20161223203901-3a8809bd8a80
github.com/pmezard/go-difflib v1.0.0
github.com/prometheus/client_golang v1.1.0 // indirect
github.com/prometheus/procfs v0.0.5 // indirect
github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect
github.com/shirou/gopsutil v2.18.12+incompatible
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.3.2
github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c // indirect
github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
golang.org/x/sys v0.0.0-20191010194322-b09406accb47
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9
golang.org/x/text v0.3.2
google.golang.org/api v0.9.0
google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 // indirect
google.golang.org/grpc v1.26.0 // indirect
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
gotest.tools/v3 v3.0.2 // indirect
k8s.io/api v0.17.3
Expand Down
73 changes: 67 additions & 6 deletions go.sum

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions pkg/minikube/cruntime/containerd.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,3 +308,8 @@ func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string {
func (r *Containerd) SystemLogCmd(len int) string {
return fmt.Sprintf("sudo journalctl -u containerd -n %d", len)
}

// Preload preloads the container runtime with k8s images
func (r *Containerd) Preload(k8sVersion string) error {
return fmt.Errorf("not yet implemented for %s", r.Name())
}
5 changes: 5 additions & 0 deletions pkg/minikube/cruntime/crio.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,3 +225,8 @@ func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string {
func (r *CRIO) SystemLogCmd(len int) string {
return fmt.Sprintf("sudo journalctl -u crio -n %d", len)
}

// Preload preloads the container runtime with k8s images
func (r *CRIO) Preload(k8sVersion string) error {
return fmt.Errorf("not yet implemented for %s", r.Name())
}
7 changes: 7 additions & 0 deletions pkg/minikube/cruntime/cruntime.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/assets"
"k8s.io/minikube/pkg/minikube/command"
"k8s.io/minikube/pkg/minikube/out"
)
Expand All @@ -46,6 +47,10 @@ func (cs ContainerState) String() string {
// CommandRunner is the subset of command.Runner this package consumes
type CommandRunner interface {
RunCmd(cmd *exec.Cmd) (*command.RunResult, error)
// Copy is a convenience method that runs a command to copy a file
Copy(assets.CopyableFile) error
// Remove is a convenience method that runs a command to remove a file
Remove(assets.CopyableFile) error
}

// Manager is a common interface for container runtimes
Expand Down Expand Up @@ -94,6 +99,8 @@ type Manager interface {
ContainerLogCmd(string, int, bool) string
// SystemLogCmd returns the command to return the system logs
SystemLogCmd(int) string
// Preload preloads the container runtime with k8s images
Preload(string) error
}

// Config is runtime configuration
Expand Down
9 changes: 9 additions & 0 deletions pkg/minikube/cruntime/cruntime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/assets"
"k8s.io/minikube/pkg/minikube/command"
)

Expand Down Expand Up @@ -215,6 +216,14 @@ func (f *FakeRunner) RunCmd(cmd *exec.Cmd) (*command.RunResult, error) {
}
}

func (f *FakeRunner) Copy(assets.CopyableFile) error {
return nil
}

func (f *FakeRunner) Remove(assets.CopyableFile) error {
return nil
}

// docker is a fake implementation of docker
func (f *FakeRunner) docker(args []string, _ bool) (string, error) {
switch cmd := args[0]; cmd {
Expand Down
44 changes: 44 additions & 0 deletions pkg/minikube/cruntime/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@ package cruntime
import (
"fmt"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/assets"
"k8s.io/minikube/pkg/minikube/download"
"k8s.io/minikube/pkg/minikube/out"
)

Expand Down Expand Up @@ -93,6 +97,15 @@ func (r *Docker) Enable(disOthers bool) error {
return nil
}

// Restart restarts Docker on a host
func (r *Docker) Restart() error {
c := exec.Command("sudo", "systemctl", "restart", "docker")
if _, err := r.Runner.RunCmd(c); err != nil {
return errors.Wrap(err, "restarting docker.")
}
return nil
}

// Disable idempotently disables Docker on a host
func (r *Docker) Disable() error {
c := exec.Command("sudo", "systemctl", "stop", "docker", "docker.socket")
Expand Down Expand Up @@ -252,3 +265,34 @@ func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string {
func (r *Docker) SystemLogCmd(len int) string {
return fmt.Sprintf("sudo journalctl -u docker -n %d", len)
}

// Preload preloads docker with k8s images:
// 1. Copy over the preloaded tarball into the VM
// 2. Extract the preloaded tarball to the correct directory
// 3. Remove the tarball within the VM
func (r *Docker) Preload(k8sVersion string) error {
tarballPath := download.TarballPath(k8sVersion)
dest := filepath.Join("/", "preloaded.tar.lz4")

// Copy over tarball into host
fa, err := assets.NewFileAsset(tarballPath, filepath.Dir(dest), filepath.Base(dest), "0644")
if err != nil {
return errors.Wrap(err, "getting file asset")
}
t := time.Now()
if err := r.Runner.Copy(fa); err != nil {
return errors.Wrap(err, "copying file")
}
glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds())

// extract the tarball to /var in the VM
if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil {
return errors.Wrapf(err, "extracting tarball: %s", rr.Output())
}

// remove the tarball in the VM
if err := r.Runner.Remove(fa); err != nil {
glog.Infof("error removing tarball: %v", err)
}
return r.Restart()
}
21 changes: 10 additions & 11 deletions pkg/minikube/node/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,15 @@ import (
)

// beginCacheKubernetesImages caches images required for kubernetes version in the background
func beginCacheKubernetesImages(g *errgroup.Group, imageRepository string, k8sVersion string) {
func beginCacheKubernetesImages(g *errgroup.Group, imageRepository string, k8sVersion, cRuntime string) {
if download.PreloadExists(k8sVersion, cRuntime) {
g.Go(func() error {
glog.Info("Caching tarball of preloaded images")
return download.Preload(k8sVersion, cRuntime)
})
return
}

if !viper.GetBool("cache-images") {
return
}
Expand Down Expand Up @@ -83,21 +91,12 @@ func doCacheBinaries(k8sVersion string) error {
}

// beginDownloadKicArtifacts downloads the kic image + preload tarball, returns true if preload is available
func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) bool {
func beginDownloadKicArtifacts(g *errgroup.Group) {
glog.Info("Beginning downloading kic artifacts")
g.Go(func() error {
glog.Infof("Downloading %s to local daemon", kic.BaseImage)
return image.WriteImageToDaemon(kic.BaseImage)
})

if download.PreloadExists(k8sVersion, cRuntime) {
g.Go(func() error {
glog.Info("Caching tarball of preloaded images")
return download.Preload(k8sVersion, cRuntime)
})
return true
}
return false
}

func waitDownloadKicArtifacts(g *errgroup.Group) {
Expand Down
6 changes: 6 additions & 0 deletions pkg/minikube/node/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config
if driver.BareMetal(drvName) {
disableOthers = false
}
if !driver.IsKIC(drvName) {
if err := cr.Preload(k8s.KubernetesVersion); err != nil {
glog.Infof("Failed to preload container runtime %s: %v", cr.Name(), err)
}
}

err = cr.Enable(disableOthers)
if err != nil {
exit.WithError("Failed to enable container runtime", err)
Expand Down
14 changes: 4 additions & 10 deletions pkg/minikube/node/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,14 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons
k8sVersion := mc.KubernetesConfig.KubernetesVersion
driverName := mc.Driver

// See if we can create a volume of preloaded images
// If not, pull images in the background while the VM boots.
// If using kic, make sure we download the kic base image
var kicGroup errgroup.Group
needKubernetesImages := true
if driver.IsKIC(driverName) {
// If we can download a preload tarball, it isn't necessary to pull Kubernetes images
if beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) {
needKubernetesImages = false
}
beginDownloadKicArtifacts(&kicGroup)
}

var cacheGroup errgroup.Group
if needKubernetesImages {
beginCacheKubernetesImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion)
}
beginCacheKubernetesImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion, mc.KubernetesConfig.ContainerRuntime)

// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
// Hence, saveConfig must be called before startHost, and again afterwards when we know the IP.
Expand All @@ -65,6 +58,7 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons

mRunner, preExists, machineAPI, host := startMachine(&mc, &n)
defer machineAPI.Close()

// configure the runtime (docker, containerd, crio)
cr := configureRuntimes(mRunner, driverName, mc.KubernetesConfig)
showVersionInfo(k8sVersion, cr)
Expand Down