fix(Dockerfile): use temporary busybox mount to create /kaniko directory

Makefile

@@ -92,16 +92,10 @@ k8s-executor-build-push:
 	DOCKER_BUILDKIT=1 docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
 	docker push $(REGISTRY)/executor:latest
 
-files/kaniko:
-	mkdir files/kaniko
-
-files/kaniko.tar: files/kaniko
-	chmod 777 files/kaniko
-	cd files && tar cf kaniko.tar kaniko
 
 .PHONY: images
 images: DOCKER_BUILDKIT=1
-images: files/kaniko.tar
+images: 
 	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
 	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug .
 	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:slim -f deploy/Dockerfile_slim .

deploy/Dockerfile

@@ -36,17 +36,19 @@ RUN mkdir -p /kaniko/.docker
 
 COPY . .
 RUN \
-    --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg \
-    make GOARCH=$TARGETARCH
+  --mount=type=cache,target=/root/.cache/go-build \
+  --mount=type=cache,target=/go/pkg \
+  make GOARCH=$TARGETARCH
 
 # Generate latest ca-certificates
 FROM debian:bullseye-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
 FROM scratch
 # Create kaniko directory with world write permission to allow non root run
-ADD files/kaniko.tar /
+# use musl busybox since it's staticly compiled
+RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] 
+
 COPY --from=0 /src/out/executor /kaniko/executor
 COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr
 COPY --from=0 /usr/local/bin/docker-credential-ecr-login /kaniko/docker-credential-ecr-login

deploy/Dockerfile_debug

@@ -36,27 +36,27 @@ RUN mkdir -p /kaniko/.docker
 
 COPY . .
 RUN \
-    --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg \
-    make GOARCH=$TARGETARCH && \
-    make GOARCH=$TARGETARCH out/warmer
+  --mount=type=cache,target=/root/.cache/go-build \
+  --mount=type=cache,target=/go/pkg \
+  make GOARCH=$TARGETARCH && \
+  make GOARCH=$TARGETARCH out/warmer
 
 # Generate latest ca-certificates
 FROM debian:bullseye-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
+# use musl busybox since it's staticly compiled on all platforms
+FROM busybox:musl as busybox
 FROM scratch
+# Create kaniko directory with world write permission to allow non root run
+RUN --mount=from=busybox,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] 
+
 COPY --from=0 /src/out/executor /kaniko/executor
 COPY --from=0 /src/out/warmer /kaniko/warmer
 COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr
 COPY --from=0 /usr/local/bin/docker-credential-ecr-login /kaniko/docker-credential-ecr-login
 COPY --from=0 /usr/local/bin/docker-credential-acr-env /kaniko/docker-credential-acr-env
-COPY --from=busybox:1.32.0 /bin /busybox
-# Since busybox needs some lib files which lie in /lib directory to run the executables on s390x, 
-# the below COPY command is added to address "ld64.so.1 not found" issue. This extra copy action will not 
-# happen on amd64 or arm64 platforms since /lib does not exist in amd64 or arm64 version of busybox container.
-# Similar issues could be found in https://github.com/multiarch/qemu-user-static/issues/110#issuecomment-652951564.
-COPY --from=busybox:1.32.0 /*lib /lib
+COPY --from=busybox /bin /busybox
 # Declare /busybox as a volume to get it automatically in the path to ignore
 VOLUME /busybox
 

deploy/Dockerfile_slim

@@ -31,6 +31,10 @@ FROM debian:bullseye-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
 FROM scratch
+
+# Create kaniko directory with world write permission to allow non root run
+# use musl busybox since it's staticly compiled
+RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] 
 COPY --from=0 /src/out/executor /kaniko/executor
 COPY files/nsswitch.conf /etc/nsswitch.conf
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /kaniko/ssl/certs/

deploy/Dockerfile_warmer

@@ -36,15 +36,19 @@ RUN mkdir -p /kaniko/.docker
 
 COPY . .
 RUN \
-    --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg \
-    make GOARCH=$TARGETARCH out/warmer
+  --mount=type=cache,target=/root/.cache/go-build \
+  --mount=type=cache,target=/go/pkg \
+  make GOARCH=$TARGETARCH out/warmer
 
 # Generate latest ca-certificates
 FROM debian:bullseye-slim AS certs
 RUN apt update && apt install -y ca-certificates
 
 FROM scratch
+# Create kaniko directory with world write permission to allow non root run
+# use musl busybox since it's staticly compiled
+RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] 
+
 COPY --from=0 /src/out/warmer /kaniko/warmer
 COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr
 COPY --from=0 /usr/local/bin/docker-credential-ecr-login /kaniko/docker-credential-ecr-login

scripts/minikube-setup.sh

@@ -22,41 +22,47 @@ if ! conntrack --version &>/dev/null; then
   sudo apt-get -qq -y install conntrack
 fi
 
-if ! command -v minikube; then
-  curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
-  chmod +x minikube
-  sudo mv minikube /usr/local/bin/
+# taken from https://github.com/kubernetes/minikube/blob/b45b29c5df6f88c6ac0afd60079a6190dc1e32c9/hack/jenkins/linux_integration_tests_none.sh#L38
+if ! kubeadm &>/dev/null; then
+  echo "WARNING: kubeadm is not installed. will try to install."
+  curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubeadm"
+  sudo install kubeadm /usr/local/bin/kubeadm
 fi
 
+# "none" driver specific cleanup from previous runs.
+sudo kubeadm reset -f --cri-socket unix:///var/run/cri-dockerd.sock || true
+# kubeadm reset may not stop pods immediately
+docker rm -f $(docker ps -aq) >/dev/null 2>&1 || true
 
+# always install minikube, because version inconsistency is possible and could lead to weird errors
+curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+chmod +x minikube
+sudo mv minikube /usr/local/bin/
 
 # Minikube needs cri-dockerd to run clusters 1.24+
 CRI_DOCKERD_VERSION="${CRI_DOCKERD_VERSION:-0.2.3}"
 CRI_DOCKERD_BINARY_URL="https://github.com/Mirantis/cri-dockerd/releases/download/v${CRI_DOCKERD_VERSION}/cri-dockerd-${CRI_DOCKERD_VERSION}.amd64.tgz"
 
 curl -Lo cri-dockerd.tgz $CRI_DOCKERD_BINARY_URL
 tar xfz cri-dockerd.tgz
-sudo mv cri-dockerd/cri-dockerd /usr/bin/cri-docker
+chmod +x cri-dockerd/cri-dockerd
+sudo mv cri-dockerd/cri-dockerd /usr/bin/cri-dockerd
 
 git clone https://github.com/Mirantis/cri-dockerd.git /tmp/cri-dockerd
 sudo cp /tmp/cri-dockerd/packaging/systemd/* /etc/systemd/system
 sudo systemctl daemon-reload
 sudo systemctl enable cri-docker.service
 sudo systemctl enable --now cri-docker.socket
 
-if ! command -v crictl; then
-  CRICTL_VERSION="v1.24.1"
-  curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz --output crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
-  sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin
-  rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz
-fi
+CRICTL_VERSION="v1.17.0"
+curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz --output crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
+sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin
+rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz
 
 sudo apt-get update
 sudo apt-get install -y liblz4-tool
 cat /proc/cpuinfo
 
-sudo systemctl unmask docker
-minikube start --vm-driver=none --force
+minikube start --vm-driver=none --force --addons="registry,default-storageclass,storage-provisioner" || minikube logs;
 minikube status
-minikube addons enable registry
 kubectl cluster-info