diff --git a/Makefile b/Makefile index 837be3f7e29f..1f797b7ad6a8 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions -ISO_VERSION ?= v1.23.1 +ISO_VERSION ?= v1.23.1-1633115168-12081 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta DEB_VERSION ?= $(subst -,~,$(RAW_VERSION)) DEB_REVISION ?= 0 @@ -286,7 +286,7 @@ minikube_iso: deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/b if [ ! -d $(BUILD_DIR)/buildroot ]; then \ mkdir -p $(BUILD_DIR); \ git clone --depth=1 --branch=$(BUILDROOT_BRANCH) https://github.com/buildroot/buildroot $(BUILD_DIR)/buildroot; \ - cp $(PWD)/deploy/iso/minikube-iso/go.hash $(BUILD_DIR)/buildroot/package/go/go.hash; \ + cp deploy/iso/minikube-iso/go.hash $(BUILD_DIR)/buildroot/package/go/go.hash; \ fi; $(MAKE) BR2_EXTERNAL=../../deploy/iso/minikube-iso minikube_defconfig -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) $(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) host-python diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/users b/deploy/iso/minikube-iso/board/coreos/minikube/users index cdff9ff1f997..e5ece397b952 100644 --- a/deploy/iso/minikube-iso/board/coreos/minikube/users +++ b/deploy/iso/minikube-iso/board/coreos/minikube/users @@ -1 +1 @@ -docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf,podman - +docker 1000 docker 1000 =tcuser /home/docker /bin/bash wheel,vboxsf,podman,buildkit - diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/51-buildkit.preset b/deploy/iso/minikube-iso/package/buildkit-bin/51-buildkit.preset new file mode 100644 index 000000000000..37de892c7105 --- /dev/null +++ b/deploy/iso/minikube-iso/package/buildkit-bin/51-buildkit.preset @@ -0,0 +1 @@ +disable buildkit.service diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk index cc479a30fd1d..6c54c2451f10 100644 --- a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit-bin.mk @@ -12,6 +12,10 @@ BUILDKIT_BIN_SOURCE = buildkit-$(BUILDKIT_BIN_VERSION).linux-amd64.tar.gz # https://github.com/opencontainers/runc.git BUILDKIT_RUNC_VERSION = 12644e614e25b05da6fd08a38ffa0cfe1903fdec +define BUILDKIT_BIN_USERS + - -1 buildkit -1 - - - - - +endef + define BUILDKIT_BIN_INSTALL_TARGET_CMDS $(INSTALL) -D -m 0755 \ $(@D)/buildctl \ @@ -25,6 +29,24 @@ define BUILDKIT_BIN_INSTALL_TARGET_CMDS $(INSTALL) -D -m 0755 \ $(@D)/buildkitd \ $(TARGET_DIR)/usr/sbin + $(INSTALL) -D -m 644 \ + $(BUILDKIT_BIN_PKGDIR)/buildkit.conf \ + $(TARGET_DIR)/usr/lib/tmpfiles.d/buildkit.conf + $(INSTALL) -D -m 644 \ + $(BUILDKIT_BIN_PKGDIR)/buildkitd.toml \ + $(TARGET_DIR)/etc/buildkit/buildkitd.toml +endef + +define BUILDKIT_BIN_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 644 \ + $(BUILDKIT_BIN_PKGDIR)/buildkit.service \ + $(TARGET_DIR)/usr/lib/systemd/system/buildkit.service + $(INSTALL) -D -m 644 \ + $(BUILDKIT_BIN_PKGDIR)/buildkit.socket \ + $(TARGET_DIR)/usr/lib/systemd/system/buildkit.socket + $(INSTALL) -D -m 644 \ + $(BUILDKIT_BIN_PKGDIR)/51-buildkit.preset \ + $(TARGET_DIR)/usr/lib/systemd/system-preset/51-buildkit.preset endef $(eval $(generic-package)) diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.conf b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.conf new file mode 100644 index 000000000000..006273ce91b5 --- /dev/null +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.conf @@ -0,0 +1 @@ +d /run/buildkit 0770 root buildkit diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.service b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.service new file mode 100644 index 000000000000..c1dad527ec0c --- /dev/null +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.service @@ -0,0 +1,11 @@ +[Unit] +Description=BuildKit +Requires=buildkit.socket +After=buildkit.socket +Documentation=https://github.com/moby/buildkit + +[Service] +ExecStart=/usr/sbin/buildkitd --addr fd:// + +[Install] +WantedBy=multi-user.target diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.socket b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.socket new file mode 100644 index 000000000000..776b23762718 --- /dev/null +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkit.socket @@ -0,0 +1,12 @@ +[Unit] +Description=BuildKit +Documentation=https://github.com/moby/buildkit + +[Socket] +ListenStream=%t/buildkit/buildkitd.sock +SocketMode=0660 +SocketUser=root +SocketGroup=buildkit + +[Install] +WantedBy=sockets.target diff --git a/deploy/iso/minikube-iso/package/buildkit-bin/buildkitd.toml b/deploy/iso/minikube-iso/package/buildkit-bin/buildkitd.toml new file mode 100644 index 000000000000..62158d44d72a --- /dev/null +++ b/deploy/iso/minikube-iso/package/buildkit-bin/buildkitd.toml @@ -0,0 +1,5 @@ +[worker.oci] + enabled = false +[worker.containerd] + enabled = true + namespace = "k8s.io" diff --git a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash index 36451957fa5b..9db9e5e115c0 100644 --- a/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash +++ b/deploy/iso/minikube-iso/package/crio-bin/crio-bin.hash @@ -21,5 +21,5 @@ sha256 74a4e916acddc6cf47ab5752bdebb6732ce2c028505ef57b7edc21d2da9039b6 v1.18.4. sha256 fc8a8e61375e3ce30563eeb0fd6534c4f48fc20300a72e6ff51cc99cb2703516 v1.19.0.tar.gz sha256 6165c5b8212ea03be2a465403177318bfe25a54c3e8d66d720344643913a0223 v1.19.1.tar.gz sha256 76fd7543bc92d4364a11060f43a5131893a76c6e6e9d6de3a6bb6292c110b631 v1.20.0.tar.gz -sha256 1c01d4a76cdcfe3ac24147eb1d5f6ebd782bd98fb0ac0c19b79bd5a6560b1481 v1.20.2.tar.gz +sha256 36d9f4cf4966342e2d4099e44d8156c55c6a10745c67ce4f856aa9f6dcc2d9ba v1.20.2.tar.gz sha256 bc53ea8977e252bd9812974c33ff654ee22076598e901464468c5c105a5ef773 v1.22.0.tar.gz diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index f7092a32d70b..8b91fe53d29f 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -142,14 +142,21 @@ COPY deploy/kicbase/containerd-fuse-overlayfs.service /etc/systemd/system/contai # install buildkit RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm-v7/') \ && echo "Installing buildkit ..." \ + && addgroup --system buildkit \ && export BUILDKIT_BASE_URL="https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}" \ && curl -sSL --retry 5 --output /tmp/buildkit.tgz "${BUILDKIT_BASE_URL}/buildkit-${BUILDKIT_VERSION}.linux-${ARCH}.tar.gz" \ && tar -C /usr/local -xzvf /tmp/buildkit.tgz \ && rm -rf /tmp/buildkit.tgz \ + && mkdir -p /usr/local/lib/systemd/system \ + && curl -L --retry 5 --output /usr/local/lib/systemd/system/buildkit.service "https://raw.githubusercontent.com/moby/buildkit/${BUILDKIT_VERSION}/examples/systemd/buildkit.service" \ + && curl -L --retry 5 --output /usr/local/lib/systemd/system/buildkit.socket "https://raw.githubusercontent.com/moby/buildkit/${BUILDKIT_VERSION}/examples/systemd/buildkit.socket" \ + && mkdir -p /etc/buildkit \ + && echo "[worker.oci]\n enabled = false\n[worker.containerd]\n enabled = true\n namespace = \"k8s.io\"" > /etc/buildkit/buildkitd.toml \ && chmod 755 /usr/local/bin/buildctl \ && chmod 755 /usr/local/bin/buildkit-runc \ && chmod 755 /usr/local/bin/buildkit-qemu-* \ - && chmod 755 /usr/local/bin/buildkitd + && chmod 755 /usr/local/bin/buildkitd \ + && systemctl enable buildkit.socket # Install cri-o/podman dependencies: RUN sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ @@ -210,6 +217,7 @@ EXPOSE 22 RUN adduser --ingroup docker --disabled-password --gecos '' docker RUN adduser docker sudo RUN adduser docker podman +RUN adduser docker buildkit RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER docker RUN mkdir /home/docker/.ssh diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 6c116840c8c5..e397e7375c3b 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -24,13 +24,13 @@ import ( const ( // Version is the current version of kic - Version = "v0.0.27" + Version = "v0.0.27-1633027942-12081" // SHA of the kic base image - baseImageSHA = "89b4738ee74ba28684676e176752277f0db46f57d27f0e08c3feec89311e22de" + baseImageSHA = "4780f1897569d2bf77aafb3d133a08d42b4fe61127f06fcfc90c2c5d902d893c" // The name of the GCR kicbase repository - gcrRepo = "gcr.io/k8s-minikube/kicbase" + gcrRepo = "gcr.io/k8s-minikube/kicbase-builds" // The name of the Dockerhub kicbase repository - dockerhubRepo = "docker.io/kicbase/stable" + dockerhubRepo = "docker.io/kicbase/build" ) var ( diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index ca50cd9892ef..86b202d3e618 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -393,10 +393,6 @@ func downloadRemote(cr CommandRunner, src string) (string, error) { // BuildImage builds an image into this runtime func (r *Containerd) BuildImage(src string, file string, tag string, push bool, env []string, opts []string) error { - if err := r.initBuildkitDaemon(); err != nil { - return fmt.Errorf("failed to init buildkit daemon: %v", err) - } - // download url if not already present dir, err := downloadRemote(r.Runner, src) if err != nil { @@ -456,24 +452,6 @@ func (r *Containerd) PushImage(name string) error { } return nil } -func (r *Containerd) initBuildkitDaemon() error { - // if daemon is already running, do nothing - cmd := exec.Command("pgrep", "buildkitd") - if _, err := r.Runner.RunCmd(cmd); err == nil { - return nil - } - - // otherwise, start daemon - cmd = exec.Command("/bin/bash", "-c", "sudo -b buildkitd --oci-worker false --containerd-worker true --containerd-worker-namespace k8s.io &> /dev/null") - if _, err := r.Runner.RunCmd(cmd); err != nil { - return fmt.Errorf("failed to start buildkit daemon: %v", err) - } - - // give the daemon time to finish starting up or image build will fail - time.Sleep(1 * time.Second) - - return nil -} // CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") func (r *Containerd) CGroupDriver() (string, error) { diff --git a/pkg/minikube/download/iso.go b/pkg/minikube/download/iso.go index 08f4042ba102..6370cf2e6d1d 100644 --- a/pkg/minikube/download/iso.go +++ b/pkg/minikube/download/iso.go @@ -40,7 +40,7 @@ const fileScheme = "file" // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order func DefaultISOURLs() []string { v := version.GetISOVersion() - isoBucket := "minikube/iso" + isoBucket := "minikube-builds/iso/12081" return []string{ fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", isoBucket, v), fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s.iso", v, v), diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index bcccd4091e4b..fdf64a0c61f7 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -26,7 +26,7 @@ minikube start [flags] --apiserver-names strings A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine --apiserver-port int The apiserver listening port (default 8443) --auto-update-drivers If set, automatically updates drivers to the latest version. Defaults to true. (default true) - --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.27@sha256:89b4738ee74ba28684676e176752277f0db46f57d27f0e08c3feec89311e22de") + --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.27-1633027942-12081@sha256:4780f1897569d2bf77aafb3d133a08d42b4fe61127f06fcfc90c2c5d902d893c") --cache-images If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true) --cert-expiration duration Duration until minikube certificate expiration, defaults to three years (26280h). (default 26280h0m0s) --cni string CNI plug-in to use. Valid options: auto, bridge, calico, cilium, flannel, kindnet, or path to a CNI manifest (default: auto) @@ -66,7 +66,7 @@ minikube start [flags] --insecure-registry strings Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added. --install-addons If set, install addons. Defaults to true. (default true) --interactive Allow user prompts for more information (default true) - --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube/iso/minikube-v1.23.1.iso,https://github.com/kubernetes/minikube/releases/download/v1.23.1/minikube-v1.23.1.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.23.1.iso]) + --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/12081/minikube-v1.23.1-1633115168-12081.iso,https://github.com/kubernetes/minikube/releases/download/v1.23.1-1633115168-12081/minikube-v1.23.1-1633115168-12081.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.23.1-1633115168-12081.iso]) --keep-context This will keep the existing kubectl context and will create a minikube context. --kubernetes-version string The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.22.2, 'latest' for v1.22.3-rc.0). Defaults to 'stable'. --kvm-gpu Enable experimental NVIDIA GPU support in minikube diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 997095b199af..ff9f00e7c9d1 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -255,6 +255,10 @@ func validateImageCommands(ctx context.Context, t *testing.T, profile string) { t.Run("ImageBuild", func(t *testing.T) { MaybeParallel(t) + if _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "pgrep", "buildkitd")); err == nil { + t.Errorf("buildkitd process is running, should not be running until `minikube image build` is ran") + } + newImage := fmt.Sprintf("localhost/my-image:%s", profile) // try to build the new image with minikube