Crash in plugin due to concurrent map writes #53
Description
docker-ovs-plugin version 0.2
Looks like there are a few global maps that are not protected with mutex's
docker-ovs-plugin/ovs/ovsdb.go
Line 24 in f58463c
I was running an experiment with many containers starting and stopping, the plugin panic'd with the following backtrace:
fatal error: concurrent map writes
goroutine 3521 [running]:
runtime.throw(0x8283a9, 0x15)
/usr/local/go/src/runtime/panic.go:566 +0x95 fp=0xc4202136d8 sp=0xc4202136b8
runtime.mapassign1(0x7b2e40, 0xc4200dcea0, 0xc420213830, 0xc420213860)
/usr/local/go/src/runtime/hashmap.go:458 +0x8ef fp=0xc4202137c0 sp=0xc4202136d8
github.com/gopher-net/docker-ovs-plugin/ovs.populateCache(0xc420110c60)
/go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:156 +0x307 fp=0xc420213940 sp=0xc4202137c0
github.com/gopher-net/docker-ovs-plugin/ovs.OvsdbNotifier.Update(0x78c4a0, 0xc4204278c0, 0xc420110c60)
/go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:36 +0x5b fp=0xc420213978 sp=0xc420213940
github.com/gopher-net/docker-ovs-plugin/ovs.(*OvsdbNotifier).Update(0xa0a810, 0x78c4a0, 0xc4204278c0, 0xc420110c60)
<autogenerated>:1 +0x6a fp=0xc4202139b8 sp=0xc420213978
github.com/socketplane/libovsdb.update(0xc4200de0e0, 0xc420180300, 0x2, 0x4, 0xc420500230, 0x0, 0x0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:133 +0x24b fp=0xc420213a68 sp=0xc4202139b8
runtime.call64(0xc4200dc330, 0x858e90, 0xc4201803c0, 0x2800000038)
/usr/local/go/src/runtime/asm_amd64.s:480 +0x4c fp=0xc420213ab8 sp=0xc420213a68
reflect.Value.call(0x7af420, 0x858e90, 0x13, 0x821de5, 0x4, 0xc420213ee0, 0x3, 0x3, 0x0, 0x7f7940, ...)
/usr/local/go/src/reflect/value.go:434 +0x5c8 fp=0xc420213e08 sp=0xc420213ab8
reflect.Value.Call(0x7af420, 0x858e90, 0x13, 0xc420213ee0, 0x3, 0x3, 0x0, 0x0, 0x0)
/usr/local/go/src/reflect/value.go:302 +0xa4 fp=0xc420213e70 sp=0xc420213e08
github.com/cenkalti/rpc2.(*Client).readRequest.func1(0xc4200e2080, 0xc4200de0e0, 0x78c4a0, 0xc42019e2e0, 0x197, 0x7806c0, 0xc420500230, 0x16, 0x0, 0xc42022d7c8, ...)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:151 +0x133 fp=0xc420213f38 sp=0xc420213e70
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420213f40 sp=0xc420213f38
created by github.com/cenkalti/rpc2.(*Client).readRequest
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:169 +0x25b
goroutine 1 [IO wait, 87 minutes]:
net.runtime_pollWait(0x7ff12bfc4f18, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420058bc0, 0x72, 0xc42010b8f8, 0xc420012160)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420058bc0, 0x9cd380, 0xc420012160)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420058b60, 0x0, 0x9cbc40, 0xc4201433e0)
/usr/local/go/src/net/fd_unix.go:419 +0x238
net.(*UnixListener).accept(0xc420143160, 0xc420132cf0, 0xc42010b9e0, 0x66f69f)
/usr/local/go/src/net/unixsock_posix.go:158 +0x32
net.(*UnixListener).Accept(0xc420143160, 0xc42010ba30, 0xc42010ba38, 0xc42010ba28, 0x5922dd)
/usr/local/go/src/net/unixsock.go:229 +0x49
github.com/docker/docker/pkg/listenbuffer.(*defaultListener).Accept(0xc420143180, 0x8590b8, 0xc42000f180, 0x9d0680, 0xc420132cf0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/docker/docker/pkg/listenbuffer/buffer.go:71 +0x3f
net/http.(*Server).Serve(0xc42000ef00, 0x9cfc40, 0xc420143180, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2273 +0x1ce
github.com/gopher-net/dknet.(*Handler).listenAndServe(0xc420143060, 0x822069, 0x4, 0x821cb5, 0x3, 0x822025, 0x4, 0x0, 0x0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/gopher-net/dknet/api.go:283 +0x1dc
github.com/gopher-net/dknet.(*Handler).ServeUnix(0xc420143060, 0x822025, 0x4, 0x821cb5, 0x3, 0x7, 0xc4200a0700)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/gopher-net/dknet/api.go:252 +0x68
main.Run(0xc4200a0790)
/go/src/github.com/gopher-net/docker-ovs-plugin/main.go:44 +0xb6
github.com/codegangsta/cli.(*App).Run(0xc4200ca0c0, 0xc42000c0e0, 0x2, 0x2, 0xc420063950, 0x4052eb)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/codegangsta/cli/app.go:132 +0x5f8
main.main()
/go/src/github.com/gopher-net/docker-ovs-plugin/main.go:30 +0x19b
goroutine 17 [syscall, 87 minutes, locked to thread]:
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1
goroutine 18 [IO wait]:
net.runtime_pollWait(0x7ff12bfc4fd8, 0x72, 0x3)
/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4200de0d0, 0x72, 0xc42018fba8, 0xc420012160)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4200de0d0, 0x9cd380, 0xc420012160)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4200de070, 0xc42050a000, 0xfe00, 0xfe00, 0x0, 0x9cd380, 0xc420012160)
/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc4200e0000, 0xc42050a000, 0xfe00, 0xfe00, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x70
encoding/json.(*Decoder).refill(0xc4200e4000, 0x7a98a0, 0xc42019e201)
/usr/local/go/src/encoding/json/stream.go:152 +0xfa
encoding/json.(*Decoder).readValue(0xc4200e4000, 0x0, 0x0, 0x671ccb)
/usr/local/go/src/encoding/json/stream.go:128 +0x198
encoding/json.(*Decoder).Decode(0xc4200e4000, 0x77ea40, 0xc4200e8020, 0xc420500230, 0x16)
/usr/local/go/src/encoding/json/stream.go:57 +0x8e
github.com/cenkalti/rpc2/jsonrpc.(*jsonCodec).ReadHeader(0xc4200e8000, 0xc4200c7740, 0xc4200c7760, 0x0, 0x0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/jsonrpc/jsonrpc.go:93 +0x70
github.com/cenkalti/rpc2.(*Client).readLoop(0xc4200de0e0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:79 +0x116
github.com/cenkalti/rpc2.(*Client).Run(0xc4200de0e0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/cenkalti/rpc2/client.go:54 +0x2b
created by github.com/socketplane/libovsdb.Connect
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:55 +0x631
goroutine 19 [chan receive, 87 minutes]:
github.com/socketplane/libovsdb.handleDisconnectNotification(0xc4200de0e0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:249 +0x49
created by github.com/socketplane/libovsdb.Connect
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/socketplane/libovsdb/client.go:56 +0x656
goroutine 6 [chan receive]:
github.com/gopher-net/docker-ovs-plugin/ovs.(*ovsdber).monitorBridges(0xc420132318)
/go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:122 +0x71
created by github.com/gopher-net/docker-ovs-plugin/ovs.(*ovsdber).initDBCache
/go/src/github.com/gopher-net/docker-ovs-plugin/ovs/ovsdb.go:66 +0x2b9
goroutine 7 [syscall]:
syscall.Syscall6(0x2c, 0x7, 0xc42052ee60, 0x20, 0x0, 0xc42052ee34, 0xc, 0xc420186e10, 0x10, 0xc42052ee60)
/usr/local/go/src/syscall/asm_linux_amd64.s:44 +0x5
syscall.sendto(0x7, 0xc42052ee60, 0x20, 0x20, 0x0, 0xc42052ee34, 0xc40000000c, 0xc42052ee60, 0xc420384690)
/usr/local/go/src/syscall/zsyscall_linux_amd64.go:1729 +0x80
syscall.Sendto(0x7, 0xc42052ee60, 0x20, 0x20, 0x0, 0x9cbf40, 0xc42052ee28, 0xc4204cd8f0, 0xc4204cd900)
/usr/local/go/src/syscall/syscall_unix.go:265 +0x92
github.com/vishvananda/netlink/nl.(*NetlinkSocket).Send(0xc42052ee20, 0xc420186e10, 0xc42052ee20, 0x0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/vishvananda/netlink/nl/nl_linux.go:333 +0x7c
github.com/vishvananda/netlink/nl.(*NetlinkRequest).Execute(0xc420186e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/vishvananda/netlink/nl/nl_linux.go:215 +0xfd
github.com/vishvananda/netlink.LinkDel(0x9ce200, 0xc42055ff80, 0x4, 0xc4200e27c0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/vishvananda/netlink/link_linux.go:432 +0xfe
github.com/gopher-net/docker-ovs-plugin/ovs.(*Driver).Leave(0xc420132300, 0xc420186cf0, 0xc420186cf0, 0x0)
/go/src/github.com/gopher-net/docker-ovs-plugin/ovs/driver.go:171 +0x1b6
github.com/gopher-net/dknet.(*Handler).initMux.func9(0x9cfec0, 0xc420065ee0, 0xc42034a0f0)
/go/src/github.com/gopher-net/docker-ovs-plugin/Godeps/_workspace/src/github.com/gopher-net/dknet/api.go:233 +0xb4
net/http.HandlerFunc.ServeHTTP(0xc420136dc0, 0x9cfec0, 0xc420065ee0, 0xc42034a0f0)
/usr/local/go/src/net/http/server.go:1726 +0x44
net/http.(*ServeMux).ServeHTTP(0xc420132c00, 0x9cfec0, 0xc420065ee0, 0xc42034a0f0)
/usr/local/go/src/net/http/server.go:2022 +0x7f
net/http.serverHandler.ServeHTTP(0xc42000ef00, 0x9cfec0, 0xc420065ee0, 0xc42034a0f0)
/usr/local/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc42000f180, 0x9d05c0, 0xc42012d480)
/usr/local/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2293 +0x44d