start_stop_with_containerd_runtime panic: test timed out after 30m (bootstrap-kubelet.conf: no such file or directory)

[tstromberg]

Seen recently. It's difficult to tell if this is a problem with the test, or if it's simply the addition of start_stop_with_containerd_runtime and upgrade to Kubernetes 1.12.3 conspiring to make tests slow.

=== RUN TestPersistence
--- PASS: TestPersistence (95.10s)
=== RUN TestStartStop
=== RUN TestStartStop/default
=== RUN TestStartStop/start_stop_with_containerd_runtime
panic: test timed out after 30m0s

goroutine 514 [running]:
testing.startAlarm.func1()
/usr/local/go/src/testing/testing.go:1023 +0xf9
created by time.goFunc
/usr/local/go/src/time/sleep.go:170 +0x44

goroutine 1 [chan receive, 25 minutes]:
testing.(*T).Run(0xc4201be820, 0x1f6e7af, 0xd, 0x1ff3608, 0x1059c01)
/usr/local/go/src/testing/testing.go:698 +0x2f4
testing.runTests.func1(0xc4201be820)
/usr/local/go/src/testing/testing.go:882 +0x67
testing.tRunner(0xc4201be820, 0xc4203bdde0)
/usr/local/go/src/testing/testing.go:657 +0x96
testing.runTests(0xc4200bd480, 0x27a8520, 0x4, 0x4, 0xc4203bde78)
/usr/local/go/src/testing/testing.go:888 +0x2c1
testing.(*M).Run(0xc4204b3f20, 0xc4203bdf20)
/usr/local/go/src/testing/testing.go:822 +0xfc
main.main()
k8s.io/minikube/test/integration/_test/_testmain.go:48 +0xf7

goroutine 5 [chan receive]:
k8s.io/minikube/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x27b4a00)
/var/lib/jenkins/go/src/k8s.io/minikube/vendor/github.com/golang/glog/glog.go:879 +0x7a
created by k8s.io/minikube/vendor/github.com/golang/glog.init.1
/var/lib/jenkins/go/src/k8s.io/minikube/vendor/github.com/golang/glog/glog.go:410 +0x21d

goroutine 6 [syscall, 30 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:116 +0xff
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 483 [chan receive, 21 minutes]:
testing.(*T).Run(0xc42006a1a0, 0x1f886db, 0x22, 0xc420464bd0, 0x71f01)
/usr/local/go/src/testing/testing.go:698 +0x2f4
k8s.io/minikube/test/integration.TestStartStop(0xc42006a1a0)
/var/lib/jenkins/go/src/k8s.io/minikube/test/integration/start_stop_delete_test.go:82 +0x13a
testing.tRunner(0xc42006a1a0, 0x1ff3608)
/usr/local/go/src/testing/testing.go:657 +0x96
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:697 +0x2ca

goroutine 415 [syscall, 21 minutes]:
syscall.Syscall6(0xad, 0x1, 0x1913, 0xc4201ab998, 0x24, 0x0, 0x0, 0x2b1b4b0, 0x0, 0xc4203f3360)
/usr/local/go/src/syscall/asm_darwin_amd64.s:41 +0x5
os.(*Process).blockUntilWaitable(0xc420430ae0, 0xc4200b31e0, 0xc4203f33a0, 0xb)
/usr/local/go/src/os/wait_waitid.go:28 +0xa5
os.(*Process).wait(0xc420430ae0, 0x100f542, 0xc420284370, 0xc4200882c0)
/usr/local/go/src/os/exec_unix.go:22 +0x4d
os.(*Process).Wait(0xc420430ae0, 0x0, 0x0, 0x1ff4cb8)
/usr/local/go/src/os/exec.go:115 +0x2b
os/exec.(*Cmd).Wait(0xc4200b31e0, 0x0, 0x0)
/usr/local/go/src/os/exec/exec.go:435 +0x62
os/exec.(*Cmd).Run(0xc4200b31e0, 0xc420260410, 0xa)
/usr/local/go/src/os/exec/exec.go:280 +0x5c
os/exec.(*Cmd).Output(0xc4200b31e0, 0x32, 0xc420090280, 0xa, 0xa, 0xc4200b31e0)
/usr/local/go/src/os/exec/exec.go:474 +0x11c
k8s.io/minikube/test/integration/util.(*MinikubeRunner).RunCommand(0xc4204b83c0, 0xc42009e180, 0xb9, 0x1, 0x3, 0xc42009e180)
/var/lib/jenkins/go/src/k8s.io/minikube/test/integration/util/util.go:74 +0xe5
k8s.io/minikube/test/integration/util.(*MinikubeRunner).Start(0xc4204b83c0)
/var/lib/jenkins/go/src/k8s.io/minikube/test/integration/util/util.go:130 +0x2ab
k8s.io/minikube/test/integration.TestStartStop.func1(0xc4204a6f70)
/var/lib/jenkins/go/src/k8s.io/minikube/test/integration/start_stop_delete_test.go:59 +0x27f
testing.tRunner(0xc4204a6f70, 0xc420464bd0)
/usr/local/go/src/testing/testing.go:657 +0x96
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:697 +0x2ca

goroutine 124 [select, 27 minutes]:
net/http.(*persistConn).writeLoop(0xc4201cc480)
/usr/local/go/src/net/http/transport.go:1704 +0x43a
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1118 +0xa5a

goroutine 123 [select, 27 minutes]:
net/http.(*persistConn).readLoop(0xc4201cc480)
/usr/local/go/src/net/http/transport.go:1599 +0x9ec
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1117 +0xa35

goroutine 503 [syscall, 19 minutes]:
syscall.Syscall(0x3, 0x5, 0xc420442ee9, 0xf17, 0x1f, 0x0, 0x0)
/usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0x5, 0xc420442ee9, 0xf17, 0xf17, 0x1f, 0x0, 0x0)
/usr/local/go/src/syscall/zsyscall_darwin_amd64.go:973 +0x55
syscall.Read(0x5, 0xc420442ee9, 0xf17, 0xf17, 0x1f, 0x0, 0x0)
/usr/local/go/src/syscall/syscall_unix.go:162 +0x49
os.(*File).read(0xc42048c0b0, 0xc420442ee9, 0xf17, 0xf17, 0x1f, 0x0, 0x0)
/usr/local/go/src/os/file_unix.go:165 +0x6a
os.(*File).Read(0xc42048c0b0, 0xc420442ee9, 0xf17, 0xf17, 0x1f, 0x0, 0x0)
/usr/local/go/src/os/file.go:101 +0x76
bytes.(*Buffer).ReadFrom(0xc4200a2850, 0x273efc0, 0xc42048c0b0, 0x2b87248, 0xc4200a2850, 0xc420336f01)
/usr/local/go/src/bytes/buffer.go:179 +0x160
io.copyBuffer(0x273bb40, 0xc4200a2850, 0x273efc0, 0xc42048c0b0, 0x0, 0x0, 0x0, 0xc4204f0358, 0x0, 0x0)
/usr/local/go/src/io/io.go:384 +0x2cb
io.Copy(0x273bb40, 0xc4200a2850, 0x273efc0, 0xc42048c0b0, 0x1d80ee0, 0xc4204f0300, 0xc420336fb0)
/usr/local/go/src/io/io.go:360 +0x68
os/exec.(*Cmd).writerDescriptor.func1(0x1d80ee0, 0xc4204f0300)
/usr/local/go/src/os/exec/exec.go:254 +0x4d
os/exec.(*Cmd).Start.func1(0xc4200b31e0, 0xc4203f3300)
/usr/local/go/src/os/exec/exec.go:371 +0x27
created by os/exec.(*Cmd).Start
/usr/local/go/src/os/exec/exec.go:372 +0x4e4

goroutine 504 [syscall, 19 minutes]:
syscall.Syscall(0x3, 0xa, 0xc4205be000, 0x8000, 0x369, 0x0, 0x0)
/usr/local/go/src/syscall/asm_darwin_amd64.s:16 +0x5
syscall.read(0xa, 0xc4205be000, 0x8000, 0x8000, 0x369, 0x0, 0x0)
/usr/local/go/src/syscall/zsyscall_darwin_amd64.go:973 +0x55
syscall.Read(0xa, 0xc4205be000, 0x8000, 0x8000, 0x369, 0x0, 0x0)
/usr/local/go/src/syscall/syscall_unix.go:162 +0x49
os.(*File).read(0xc42048c0c8, 0xc4205be000, 0x8000, 0x8000, 0x369, 0x0, 0x0)
/usr/local/go/src/os/file_unix.go:165 +0x6a
os.(*File).Read(0xc42048c0c8, 0xc4205be000, 0x8000, 0x8000, 0x369, 0x0, 0x0)
/usr/local/go/src/os/file.go:101 +0x76
io.copyBuffer(0x273f1c0, 0xc420260410, 0x273efc0, 0xc42048c0c8, 0xc4205be000, 0x8000, 0x8000, 0xc4203fa3b8, 0x0, 0x0)
/usr/local/go/src/io/io.go:390 +0x100
io.Copy(0x273f1c0, 0xc420260410, 0x273efc0, 0xc42048c0c8, 0x1d80ee0, 0xc4203fa360, 0xc4203eb7b0)
/usr/local/go/src/io/io.go:360 +0x68
os/exec.(*Cmd).writerDescriptor.func1(0x1d80ee0, 0xc4203fa360)
/usr/local/go/src/os/exec/exec.go:254 +0x4d
os/exec.(*Cmd).Start.func1(0xc4200b31e0, 0xc4203f33a0)
/usr/local/go/src/os/exec/exec.go:371 +0x27
created by os/exec.(*Cmd).Start
/usr/local/go/src/os/exec/exec.go:372 +0x4e4

[tstromberg]

Also worth mentioning that the kubelet is crashing until the very end with:

Dec 06 13:26:41 minikube kubelet[4186]: I1206 13:26:41.937068 4186 server.go:408] Version: 
v1.12.3

Dec 06 13:26:41 minikube kubelet[4186]: F1206 13:26:41.937400 4186 server.go:262] 
failed to run Kubelet:
unable to load bootstrap kubeconfig: stat /etc/kubernetes/bootstrap-kubelet.conf:
no such file or directory

Dec 06 13:26:41 minikube systemd[1]: kubelet.service: Main process exited, code=exited,
 status=255/n/a

[priyawadhwa]

Ref #2927 #2703