[0.9.4.1] panic: interface conversion: interface {} is map[string]interface {}, not *influxql.meanMapOutput

[ecables]

Two nodes in a 3-node cluster panic'd with the following panic message.

panic: interface conversion: interface {} is map[string]interface {}, not *influxql.meanMapOutput

goroutine 85082 [running]:
github.com/influxdb/influxdb/influxql.ReduceMean(0xc881593100, 0xe, 0x10, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/influxql/functions.go:565 +0x1af
github.com/influxdb/influxdb/tsdb.(*SelectExecutor).executeAggregate(0xc87488d530, 0xc8c9c245a0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/executor.go:522 +0x16cd
created by github.com/influxdb/influxdb/tsdb.(*SelectExecutor).Execute
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/executor.go:96 +0xf2

goroutine 1 [chan receive, 89 minutes]:
main.(*Main).Run(0xc8a0cabf10, 0xc82000a0b0, 0x4, 0x4, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/main.go:91 +0xa16
main.main()
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/main.go:42 +0x29b

goroutine 17 [syscall, 91 minutes, locked to thread]:
runtime.goexit()
    /home/philip/.gvm/gos/go1.5/src/runtime/asm_amd64.s:1696 +0x1

goroutine 5 [syscall, 91 minutes]:
os/signal.loop()
    /home/philip/.gvm/gos/go1.5/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
    /home/philip/.gvm/gos/go1.5/src/os/signal/signal_unix.go:28 +0x37

goroutine 20 [chan receive, 91 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820142030, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/meta.(*raftLayer).Accept(0xc820140c60, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/meta/store.go:2059 +0x51
github.com/hashicorp/raft.(*NetworkTransport).listen(0xc8201582a0)
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:346 +0x79
created by github.com/hashicorp/raft.NewNetworkTransport
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:138 +0x284

goroutine 19 [IO wait]:
net.runtime_pollWait(0x7fe8962c19f0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc820158290, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820158290, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc820158230, 0x0, 0x7fe8962c1c30, 0xc8562656c0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820142028, 0xc8202ecec8, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc820142028, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/tcpsock_posix.go:264 +0x3d
github.com/influxdb/influxdb/tcp.(*Mux).Serve(0xc820146280, 0x7fe894267368, 0xc820142028, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:48 +0x4b
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:345 +0xae9

goroutine 21 [select]:
github.com/hashicorp/raft.(*Raft).runFollower(0xc8202d2000)
    /tmp/build/src/github.com/hashicorp/raft/raft.go:613 +0x91d
github.com/hashicorp/raft.(*Raft).run(0xc8202d2000)
    /tmp/build/src/github.com/hashicorp/raft/raft.go:598 +0x91
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.run)-fm()
    /tmp/build/src/github.com/hashicorp/raft/raft.go:252 +0x20
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc8202d2000, 0xc82013ce70)
    /tmp/build/src/github.com/hashicorp/raft/state.go:152 +0x44
created by github.com/hashicorp/raft.(*raftState).goFunc
    /tmp/build/src/github.com/hashicorp/raft/state.go:153 +0x4d

goroutine 22 [select, 30 minutes]:
github.com/hashicorp/raft.(*Raft).runFSM(0xc8202d2000)
    /tmp/build/src/github.com/hashicorp/raft/raft.go:509 +0x10eb
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.runFSM)-fm()
    /tmp/build/src/github.com/hashicorp/raft/raft.go:253 +0x20
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc8202d2000, 0xc82013ce80)
    /tmp/build/src/github.com/hashicorp/raft/state.go:152 +0x44
created by github.com/hashicorp/raft.(*raftState).goFunc
    /tmp/build/src/github.com/hashicorp/raft/state.go:153 +0x4d

goroutine 23 [select]:
github.com/hashicorp/raft.(*Raft).runSnapshots(0xc8202d2000)
    /tmp/build/src/github.com/hashicorp/raft/raft.go:1636 +0x493
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.runSnapshots)-fm()
    /tmp/build/src/github.com/hashicorp/raft/raft.go:254 +0x20
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc8202d2000, 0xc82013ce90)
    /tmp/build/src/github.com/hashicorp/raft/state.go:152 +0x44
created by github.com/hashicorp/raft.(*raftState).goFunc
    /tmp/build/src/github.com/hashicorp/raft/state.go:153 +0x4d

goroutine 24 [select, 91 minutes]:
github.com/influxdb/influxdb/meta.(*localRaft).logLeaderChanges(0xc82014c280)
    /tmp/build/src/github.com/influxdb/influxdb/meta/state.go:187 +0x4f5
created by github.com/influxdb/influxdb/meta.(*localRaft).open
    /tmp/build/src/github.com/influxdb/influxdb/meta/state.go:177 +0x1344

goroutine 25 [chan receive, 91 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820142038, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/meta.(*Store).serveExecListener(0xc8200b4780)
    /tmp/build/src/github.com/influxdb/influxdb/meta/store.go:622 +0xa2
created by github.com/influxdb/influxdb/meta.(*Store).Open
    /tmp/build/src/github.com/influxdb/influxdb/meta/store.go:239 +0x364

goroutine 26 [chan receive, 91 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820142040, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/meta.(*Store).serveRPCListener(0xc8200b4780)
    /tmp/build/src/github.com/influxdb/influxdb/meta/store.go:726 +0xa2
created by github.com/influxdb/influxdb/meta.(*Store).Open
    /tmp/build/src/github.com/influxdb/influxdb/meta/store.go:242 +0x3b6

goroutine 35 [IO wait]:
net.runtime_pollWait(0x7fe8962c1ab0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc820314060, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820314060, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc820314000, 0xc820324000, 0x1000, 0x1000, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc820318000, 0xc820324000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
bufio.(*Reader).fill(0xc820320060)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).ReadByte(0xc820320060, 0x401, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:229 +0x7a
github.com/hashicorp/raft.(*NetworkTransport).handleCommand(0xc8201582a0, 0xc820320060, 0xc8203200c0, 0xc820314070, 0x0, 0x0)
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:386 +0x45
github.com/hashicorp/raft.(*NetworkTransport).handleConn(0xc8201582a0, 0x7fe8940a7000, 0xc820318000)
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:370 +0x361
created by github.com/hashicorp/raft.(*NetworkTransport).listen
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:357 +0x3a9

goroutine 37 [IO wait]:
net.runtime_pollWait(0x7fe8962c1930, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc820314370, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820314370, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc820314310, 0xc820339000, 0x1000, 0x1000, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc820318010, 0xc820339000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
bufio.(*Reader).fill(0xc8203202a0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).ReadByte(0xc8203202a0, 0x401, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:229 +0x7a
github.com/hashicorp/raft.(*NetworkTransport).handleCommand(0xc8201582a0, 0xc8203202a0, 0xc820320300, 0xc820314380, 0x0, 0x0)
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:386 +0x45
github.com/hashicorp/raft.(*NetworkTransport).handleConn(0xc8201582a0, 0x7fe8940a7000, 0xc820318010)
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:370 +0x361
created by github.com/hashicorp/raft.(*NetworkTransport).listen
    /tmp/build/src/github.com/hashicorp/raft/net_transport.go:357 +0x3a9

goroutine 50 [select, 91 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Server).monitorErrorChan(0xc820187320, 0xc820014720)
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:491 +0x145
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:351 +0xc67

goroutine 14 [select, 2 minutes]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc82015a000, 0xc820386360)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 42 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352460, 0xc8347fbe00)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 98 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203520e0, 0xc8311724e0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 115 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203521c0, 0xc830d036e0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 70 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203522a0, 0xc831f721e0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 71 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352380, 0xc850036000)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 84681 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fe884490180, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86c5c2ca0, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86c5c2ca0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86c5c2c40, 0xc8930ac000, 0x1000, 0x1000, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc896d791d0, 0xc8930ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
net/http.(*liveSwitchReader).Read(0xc8897eef68, 0xc8930ac000, 0x1000, 0x1000, 0xc8930ad000, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:219 +0xa4
io.(*LimitedReader).Read(0xc834ae5760, 0xc8930ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:427 +0xbd
bufio.(*Reader).fill(0xc87ef33080)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc87ef33080, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:132 +0xcc
net/http.(*conn).readRequest(0xc8897eef20, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:629 +0x2bb
net/http.(*conn).serve(0xc8897eef20)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1319 +0x727
created by net/http.(*Server).Serve
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1910 +0x3f6

goroutine 133 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352540, 0xc841ccaea0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 258 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352620, 0xc859560060)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 259 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352700, 0xc833bba8a0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 234 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203527e0, 0xc85d7cc120)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 235 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203528c0, 0xc8408f61e0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 134 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203529a0, 0xc83f8d3320)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 135 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352a80, 0xc8322951a0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 136 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352b60, 0xc85346a1e0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 137 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352c40, 0xc829cfb260)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 236 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352d20, 0xc86297b1a0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 237 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352e00, 0xc84c9a9680)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 260 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352ee0, 0xc867014960)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 261 [select, 1 minutes]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc83a8145a0, 0xc8621d4fc0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 262 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820352fc0, 0xc867ea4d80)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 280 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8203530a0, 0xc85b00d320)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 143 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820353180, 0xc83b71c0c0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 144 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820353260, 0xc867f68f60)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 145 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820353340, 0xc8874f9560)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 81808 [IO wait]:
net.runtime_pollWait(0x7fd6b3256f88, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8a07e4220, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8a07e4220, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8a07e41c0, 0xc8864151b0, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc870a8a010, 0xc8864151b0, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc870a8a010, 0xc8864151b0, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc870a8a010, 0xc8864151b0, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc870a8a010, 0xc870a8a010, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc870a8a010)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc870a8a010)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 82637 [select, 1 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc87f519aa0, 0xc82000d1f0, 0x7fe8940a7000, 0xc89ef8a008)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 82214 [select, 2 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc8952e6c60, 0xc82000d1f0, 0x7fe8940a7000, 0xc87ef2a080)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 284 [runnable]:
encoding/json.stateInString(0xc89c7664d0, 0x61, 0x0)
    /home/philip/.gvm/gos/go1.5/src/encoding/json/scanner.go:331
encoding/json.checkValid(0xc8ca20c000, 0x2cf6475, 0x2cf6475, 0xc89c7664d0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/encoding/json/scanner.go:24 +0xd7
encoding/json.Unmarshal(0xc8ca20c000, 0x2cf6475, 0x2cf6475, 0x993380, 0xc8c5b7d118, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/encoding/json/decode.go:79 +0x73
github.com/influxdb/influxdb/tsdb/engine/bz1.(*Engine).readSeries(0xc8540e2140, 0xc870477ee0, 0xc8884be828, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/bz1/bz1.go:347 +0x271
github.com/influxdb/influxdb/tsdb/engine/bz1.(*Engine).writeNewSeries(0xc8540e2140, 0xc870477ee0, 0xc8bc33b000, 0x4db, 0x600, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/bz1/bz1.go:312 +0x69
github.com/influxdb/influxdb/tsdb/engine/bz1.(*Engine).WriteIndex.func1(0xc870477ee0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/bz1/bz1.go:241 +0x87
github.com/boltdb/bolt.(*DB).Update(0xc8693c24e0, 0xc8901d1a78, 0x0, 0x0)
    /tmp/build/src/github.com/boltdb/bolt/db.go:547 +0xb3
github.com/influxdb/influxdb/tsdb/engine/bz1.(*Engine).WriteIndex(0xc8540e2140, 0x0, 0xc87484ba40, 0xc8bc33b000, 0x4db, 0x600, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/bz1/bz1.go:254 +0x86
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).flushMetadata(0xc820353420, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:688 +0x8d4
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820353420, 0xc8856fb560)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:616 +0x3be
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 285 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc82015e9a0, 0xc89bbe1020)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 286 [select]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc885fe6000, 0xc89966f2c0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 287 [select]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc885fe60a0, 0xc8854ccf60)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 288 [select]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc885fe6140, 0xc8529f0840)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 289 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8a78e6b60, 0xc845e560c0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 471 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8a78e6c40, 0xc89bc89500)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 472 [select]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc88610c000, 0xc8825fe060)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 82636 [IO wait]:
net.runtime_pollWait(0x7fe79021be60, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc87f3eeb50, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc87f3eeb50, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc87f3eeaf0, 0xc869c54f40, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc89ef8a008, 0xc869c54f40, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc89ef8a008, 0xc869c54f40, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc89ef8a008, 0xc869c54f40, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc89ef8a008, 0xc89ef8a008, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc89ef8a008)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc89ef8a008)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 459 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc898b64360, 0xc82000d1f0, 0x7fe8940a7000, 0xc8ac8b4000)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 1257 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc8983d2480, 0xc82000d1f0, 0x7fe8940a7000, 0xc893142000)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 1195 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc898995680, 0xc82000d1f0, 0x7fe8940a7000, 0xc820143c00)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 1197 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc898995740, 0xc82000d1f0, 0x7fe8940a7000, 0xc820143c10)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 1199 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc898995800, 0xc82000d1f0, 0x7fe8940a7000, 0xc820143c28)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 1179 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc881222300, 0xc82000d1f0, 0x7fe8940a7000, 0xc820143c90)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 605 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc820353500, 0xc82a7d9080)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 606 [select]:
github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).autoflusher(0xc852ec6000, 0xc8a4c0e540)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:469 +0x3c7
created by github.com/influxdb/influxdb/tsdb/engine/b1.(*Engine).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/b1/b1.go:134 +0x461

goroutine 607 [select]:
github.com/influxdb/influxdb/services/hh.(*Service).retryWrites(0xc8200aa750)
    /tmp/build/src/github.com/influxdb/influxdb/services/hh/service.go:103 +0x314
created by github.com/influxdb/influxdb/services/hh.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/hh/service.go:67 +0x210

goroutine 608 [select, 29 minutes]:
github.com/influxdb/influxdb/services/hh.(*Service).expireWrites(0xc8200aa750)
    /tmp/build/src/github.com/influxdb/influxdb/services/hh/service.go:121 +0x2d1
created by github.com/influxdb/influxdb/services/hh.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/hh/service.go:68 +0x232

goroutine 609 [chan receive]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820142048, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/cluster.(*Service).serve(0xc82000d1f0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:84 +0xdb
created by github.com/influxdb/influxdb/cluster.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:61 +0x135

goroutine 642 [select, 9 minutes]:
github.com/influxdb/influxdb/services/precreator.(*Service).runPrecreation(0xc82019e480)
    /tmp/build/src/github.com/influxdb/influxdb/services/precreator/service.go:75 +0x44f
created by github.com/influxdb/influxdb/services/precreator.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/precreator/service.go:53 +0x21f

goroutine 643 [chan receive, 89 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820142050, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/services/snapshotter.(*Service).serve(0xc8202a9a40)
    /tmp/build/src/github.com/influxdb/influxdb/services/snapshotter/service.go:75 +0xa9
created by github.com/influxdb/influxdb/services/snapshotter.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/snapshotter/service.go:48 +0x130

goroutine 644 [chan receive, 89 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820142058, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/services/copier.(*Service).serve(0xc82019e500)
    /tmp/build/src/github.com/influxdb/influxdb/services/copier/service.go:78 +0xa9
created by github.com/influxdb/influxdb/services/copier.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/copier/service.go:51 +0x130

goroutine 81809 [select, 2 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc8ca07b4a0, 0xc82000d1f0, 0x7fe8940a7000, 0xc870a8a010)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 645 [IO wait, 89 minutes]:
net.runtime_pollWait(0x7fe7a42c8b08, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc88454efb0, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc88454efb0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc88454ef50, 0x0, 0x7fe8962c1c30, 0xc855cda280)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc8c8d94098, 0x455470, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc8c8d94098, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/tcpsock_posix.go:264 +0x3d
net/http.(*Server).Serve(0xc8a49b00c0, 0x7fe894267368, 0xc8c8d94098, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1887 +0xb3
net/http.Serve(0x7fe894267368, 0xc8c8d94098, 0x7fd8335cf698, 0xc8715760c0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1765 +0x85
github.com/influxdb/influxdb/services/admin.(*Service).serve(0xc8202a9b30)
    /tmp/build/src/github.com/influxdb/influxdb/services/admin/service.go:107 +0x12f
created by github.com/influxdb/influxdb/services/admin.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/admin/service.go:70 +0x491

goroutine 646 [select]:
github.com/influxdb/influxdb/services/continuous_querier.(*Service).backgroundLoop(0xc8202ac6c0)
    /tmp/build/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:192 +0x565
created by github.com/influxdb/influxdb/services/continuous_querier.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:126 +0x2a4

goroutine 647 [IO wait]:
net.runtime_pollWait(0x7fe7a42c8a48, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc88454f020, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc88454f020, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc88454efc0, 0x0, 0x7fe8962c1c30, 0xc8644c91c0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc8c8d940a0, 0xc8c80b8dc0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc8c8d940a0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/tcpsock_posix.go:264 +0x3d
net/http.(*Server).Serve(0xc8a49b0000, 0x7fe894267368, 0xc8c8d940a0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1887 +0xb3
net/http.Serve(0x7fe894267368, 0xc8c8d940a0, 0x7fd8335cf5f8, 0xc8200b9400, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1765 +0x85
github.com/influxdb/influxdb/services/httpd.(*Service).serve(0xc820014d80)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/service.go:133 +0x6e
created by github.com/influxdb/influxdb/services/httpd.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/service.go:101 +0x6ca

goroutine 648 [select, 29 minutes]:
github.com/influxdb/influxdb/services/retention.(*Service).deleteShardGroups(0xc8202a9cc0)
    /tmp/build/src/github.com/influxdb/influxdb/services/retention/service.go:69 +0x351
created by github.com/influxdb/influxdb/services/retention.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/retention/service.go:45 +0x1b4

goroutine 649 [select, 29 minutes]:
github.com/influxdb/influxdb/services/retention.(*Service).deleteShards(0xc8202a9cc0)
    /tmp/build/src/github.com/influxdb/influxdb/services/retention/service.go:102 +0x7a0
created by github.com/influxdb/influxdb/services/retention.(*Service).Open
    /tmp/build/src/github.com/influxdb/influxdb/services/retention/service.go:46 +0x1d9

goroutine 650 [chan receive, 89 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Server).startServerReporting(0xc820187320)
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:442 +0x1eb
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open.func1
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:378 +0x1265

goroutine 651 [select, 89 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Command).monitorServerErrors(0xc8200b8200)
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go:138 +0x2c5
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Command).Run
    /tmp/build/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go:120 +0xeeb

goroutine 652 [select, 89 minutes, locked to thread]:
runtime.gopark(0xd19ec8, 0xc899f2c728, 0xbd5940, 0x6, 0x431018, 0x2)
    /home/philip/.gvm/gos/go1.5/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc899f2c728, 0x0, 0x18)
    /home/philip/.gvm/gos/go1.5/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc899f2c728)
    /home/philip/.gvm/gos/go1.5/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
    /home/philip/.gvm/gos/go1.5/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
    /home/philip/.gvm/gos/go1.5/src/runtime/asm_amd64.s:1696 +0x1

goroutine 81304 [IO wait]:
net.runtime_pollWait(0x7fe79021b620, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc883478e60, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc883478e60, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc883478e00, 0xc89b1dbda0, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc8b375a058, 0xc89b1dbda0, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc8b375a058, 0xc89b1dbda0, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc8b375a058, 0xc89b1dbda0, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc8b375a058, 0xc8b375a058, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc8b375a058)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc8b375a058)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 84931 [chan receive]:
github.com/influxdb/influxdb/services/httpd.(*Handler).serveQuery(0xc8200b9400, 0x7fe7907dd5e0, 0xc85635d240, 0xc895837960, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:286 +0xc2c
github.com/influxdb/influxdb/services/httpd.(*Handler).(github.com/influxdb/influxdb/services/httpd.serveQuery)-fm(0x7fe7907dd5e0, 0xc85635d240, 0xc895837960, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:98 +0x48
github.com/influxdb/influxdb/services/httpd.authenticate.func1(0x7fe7907dd5e0, 0xc85635d240, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:673 +0x70
net/http.HandlerFunc.ServeHTTP(0xc8202a7fc0, 0x7fe7907dd5e0, 0xc85635d240, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/influxdb/influxdb/services/httpd.gzipFilter.func1(0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:734 +0x2af
net/http.HandlerFunc.ServeHTTP(0xc8202a7fe0, 0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/influxdb/influxdb/services/httpd.versionHeader.func1(0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:743 +0xc0
net/http.HandlerFunc.ServeHTTP(0xc8202ae000, 0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/influxdb/influxdb/services/httpd.cors.func1(0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:776 +0x328
net/http.HandlerFunc.ServeHTTP(0xc8202ae020, 0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/influxdb/influxdb/services/httpd.requestID.func1(0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:786 +0x15d
net/http.HandlerFunc.ServeHTTP(0xc8202ae040, 0x7fe7907dd508, 0xc85635d200, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/influxdb/influxdb/services/httpd.logging.func1(0x7fe7907dd508, 0xc85635d1e0, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:794 +0x10b
net/http.HandlerFunc.ServeHTTP(0xc8202ae060, 0x7fe7907dd508, 0xc85635d1e0, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/influxdb/influxdb/services/httpd.recovery.func1(0x7fe7907dd4d0, 0xc870e5ac60, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:804 +0x114
net/http.HandlerFunc.ServeHTTP(0xc8202ae080, 0x7fe7907dd4d0, 0xc870e5ac60, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1422 +0x3a
github.com/bmizerany/pat.(*PatternServeMux).ServeHTTP(0xc82028e680, 0x7fe7907dd4d0, 0xc870e5ac60, 0xc895837960)
    /tmp/build/src/github.com/bmizerany/pat/mux.go:109 +0x244
github.com/influxdb/influxdb/services/httpd.(*Handler).ServeHTTP(0xc8200b9400, 0x7fe7907dd4d0, 0xc870e5ac60, 0xc895837960)
    /tmp/build/src/github.com/influxdb/influxdb/services/httpd/handler.go:172 +0x357
net/http.serverHandler.ServeHTTP(0xc8a49b0000, 0x7fe7907dd4d0, 0xc870e5ac60, 0xc895837960)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1862 +0x19e
net/http.(*conn).serve(0xc8a99ddad0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1361 +0xbee
created by net/http.(*Server).Serve
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1910 +0x3f6

goroutine 80883 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8ba700a80, 0xc85f339140)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 84676 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fe79021bfe0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86c0d76b0, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86c0d76b0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86c0d7650, 0xc89da8f000, 0x1000, 0x1000, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc896d791c8, 0xc89da8f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
net/http.(*liveSwitchReader).Read(0xc8a9c06e08, 0xc89da8f000, 0x1000, 0x1000, 0xc8ac206000, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:219 +0xa4
io.(*LimitedReader).Read(0xc83502bc60, 0xc89da8f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:427 +0xbd
bufio.(*Reader).fill(0xc87a828780)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc87a828780, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:132 +0xcc
net/http.(*conn).readRequest(0xc8a9c06dc0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:629 +0x2bb
net/http.(*conn).serve(0xc8a9c06dc0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1319 +0x727
created by net/http.(*Server).Serve
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1910 +0x3f6

goroutine 980 [IO wait]:
net.runtime_pollWait(0x7fe884490600, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc887058060, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc887058060, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc887058000, 0xc867c80d90, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc8ac8b4000, 0xc867c80d90, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc8ac8b4000, 0xc867c80d90, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc8ac8b4000, 0xc867c80d90, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc8ac8b4000, 0xc8ac8b4000, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc8ac8b4000)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc8ac8b4000)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 983 [IO wait]:
net.runtime_pollWait(0x7fe884490480, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86e530060, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86e530060, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86e530000, 0xc867c80e10, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc893142000, 0xc867c80e10, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc893142000, 0xc867c80e10, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc893142000, 0xc867c80e10, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc893142000, 0xc893142000, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc893142000)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc893142000)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 987 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fe8844900c0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86f550ae0, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86f550ae0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86f550a80, 0xc86eb1ec00, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc820143c00, 0xc86eb1ec00, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc820143c00, 0xc86eb1ec00, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc820143c00, 0xc86eb1ec00, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc820143c00, 0xc820143c00, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c00)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c00)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 992 [IO wait]:
net.runtime_pollWait(0x7fe88448fdc0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86f550c30, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86f550c30, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86f550bd0, 0xc874ee7530, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc820143c10, 0xc874ee7530, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc820143c10, 0xc874ee7530, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc820143c10, 0xc874ee7530, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc820143c10, 0xc820143c10, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c10)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c10)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 993 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fe88448fc40, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8a08ac140, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8a08ac140, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8a08ac0e0, 0xc858c24690, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc881c90010, 0xc858c24690, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc881c90010, 0xc858c24690, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc881c90010, 0xc858c24690, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc881c90010, 0xc881c90010, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc881c90010)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc881c90010)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 995 [IO wait]:
net.runtime_pollWait(0x7fe88448fac0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86f550df0, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86f550df0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86f550d90, 0xc8567611e0, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc820143c28, 0xc8567611e0, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc820143c28, 0xc8567611e0, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc820143c28, 0xc8567611e0, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc820143c28, 0xc820143c28, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c28)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c28)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 45429 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fd6b3257408, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8742d24c0, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8742d24c0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8742d2460, 0xc88727d000, 0x1000, 0x1000, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc89ffaa0f8, 0xc88727d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
net/http.(*liveSwitchReader).Read(0xc8c82e3178, 0xc88727d000, 0x1000, 0x1000, 0xc88669c000, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:219 +0xa4
io.(*LimitedReader).Read(0xc840a4ad60, 0xc88727d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:427 +0xbd
bufio.(*Reader).fill(0xc87a6e9080)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc87a6e9080, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:132 +0xcc
net/http.(*conn).readRequest(0xc8c82e3130, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:629 +0x2bb
net/http.(*conn).serve(0xc8c82e3130)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1319 +0x727
created by net/http.(*Server).Serve
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1910 +0x3f6

goroutine 1091 [IO wait]:
net.runtime_pollWait(0x7fe79021c160, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc86f551870, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc86f551870, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc86f551810, 0xc86321aff0, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc820143c90, 0xc86321aff0, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc820143c90, 0xc86321aff0, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc820143c90, 0xc86321aff0, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc820143c90, 0xc820143c90, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c90)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc820143c90)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 85122 [runnable]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc8c9d37380, 0xc82000d1f0, 0x7fe8940a7000, 0xc8c748e478)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 561 [select, 89 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc8967864e0, 0xc82000d1f0, 0x7fe8940a7000, 0xc881c90010)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

goroutine 80909 [select]:
github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher(0xc8b3e847e0, 0xc8b348e5a0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604 +0x567
created by github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).Open
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:240 +0x627

goroutine 85081 [chan receive]:
github.com/influxdb/influxdb/tsdb.(*QueryExecutor).executeSelectStatement(0xc82000adc0, 0x0, 0xc8874f2480, 0xc8c9c24540, 0x2710, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/query_executor.go:285 +0xeb
github.com/influxdb/influxdb/tsdb.(*QueryExecutor).ExecuteQuery.func1(0xc85635d420, 0xc870542b4e, 0x7, 0xc82000adc0, 0xc8c9c24540, 0x2710)
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/query_executor.go:159 +0x757
created by github.com/influxdb/influxdb/tsdb.(*QueryExecutor).ExecuteQuery
    /tmp/build/src/github.com/influxdb/influxdb/tsdb/query_executor.go:213 +0x9b

goroutine 82213 [IO wait]:
net.runtime_pollWait(0x7fe8844906c0, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8ac49dc60, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8ac49dc60, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8ac49dc00, 0xc8678c8a40, 0x1, 0x1, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc87ef2a080, 0xc8678c8a40, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
io.ReadAtLeast(0x7fe8940a7060, 0xc87ef2a080, 0xc8678c8a40, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:298 +0xe6
io.ReadFull(0x7fe8940a7060, 0xc87ef2a080, 0xc8678c8a40, 0x1, 0x1, 0x7fe8940a7100, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:316 +0x62
github.com/influxdb/influxdb/cluster.ReadTLV(0x7fe8940a7060, 0xc87ef2a080, 0xc87ef2a080, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:307 +0xc5
github.com/influxdb/influxdb/cluster.(*Service).handleConn(0xc82000d1f0, 0x7fe8940a7000, 0xc87ef2a080)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:135 +0x25c
github.com/influxdb/influxdb/cluster.(*Service).serve.func1(0xc82000d1f0, 0x7fe8940a7000, 0xc87ef2a080)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x6a
created by github.com/influxdb/influxdb/cluster.(*Service).serve
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:99 +0x3a9

goroutine 84831 [IO wait]:
net.runtime_pollWait(0x7fe7ac6ae740, 0x72, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8bb878610, 0x72, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8bb878610, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8bb8785b0, 0xc8881d1000, 0x1000, 0x1000, 0x0, 0x7fe8962ab050, 0xc820010140)
    /home/philip/.gvm/gos/go1.5/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc870a8a040, 0xc8881d1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/net.go:172 +0xe4
net/http.(*liveSwitchReader).Read(0xc8c82e20f8, 0xc8881d1000, 0x1000, 0x1000, 0xc885780010, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:219 +0xa4
io.(*LimitedReader).Read(0xc89d820b40, 0xc8881d1000, 0x1000, 0x1000, 0x1fe, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/io/io.go:427 +0xbd
bufio.(*Reader).fill(0xc86fe68ea0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).ReadSlice(0xc86fe68ea0, 0xc81d3cfb0a, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:328 +0x21a
bufio.(*Reader).ReadLine(0xc86fe68ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/bufio/bufio.go:357 +0x53
net/textproto.(*Reader).readLineSlice(0xc899ca3ce0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/textproto/reader.go:55 +0x81
net/textproto.(*Reader).ReadLine(0xc899ca3ce0, 0x0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/textproto/reader.go:36 +0x40
net/http.ReadRequest(0xc86fe68ea0, 0xc878608b60, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/request.go:653 +0xb6
net/http.(*conn).readRequest(0xc8c82e20b0, 0x0, 0x0, 0x0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:633 +0x32f
net/http.(*conn).serve(0xc8c82e20b0)
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1319 +0x727
created by net/http.(*Server).Serve
    /home/philip/.gvm/gos/go1.5/src/net/http/server.go:1910 +0x3f6

goroutine 81305 [select, 3 minutes]:
github.com/influxdb/influxdb/cluster.(*Service).handleConn.func1(0xc89e4b2780, 0xc82000d1f0, 0x7fe8940a7000, 0xc8b375a058)
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:122 +0xe8
created by github.com/influxdb/influxdb/cluster.(*Service).handleConn
    /tmp/build/src/github.com/influxdb/influxdb/cluster/service.go:127 +0xc3

[otoolep]

@ecables -- just to confirm, this data should be fully replicated, correct?

[otoolep]

It looks like I may have duped this, I can bring out a panic with an identical trace in response to queries.

[otoolep]

OK, I am working a hypothesis here.

I believe your data is fully replicated, but occasionally when a new shard group is created, the system (due to the nature of the system) "sees" those new shards on remote nodes before it sees that the data is also local. This triggers a remote access to that shard.

Secondly, there appears to be bug in our marshalling code that takes query output, and sends it over the network. It is this second issue I am debugging right now, as it is this issue that causes the panic.

cc @dgnorton @benbjohnson @jwilder @corylanou

[otoolep]

@dgnorton -- something is going wrong here, and I am not sure if it is related to the refactoring you did recently, or this issue was always here. The types emitted by the Mappers is getting "lost", and translated to simple maps. This is causing the receiving node to panic.

That's what it looks like to me anyway.

[otoolep]

#4174 may allow an effective workaround for this issue, where clusters are fully replicated.

[otoolep]

@dgnorton - after further examination of the code, I think this issue may predate DQ work even, and that the model for marshalling Mapper data across the network may need to be reworked.

#4174 should provide an effective workaround for the remote mapping operation in the first place (though it still needs to be established why remote mapping is occasionally occurring on a fully-replicated cluster), but it looks to me we may need to revisit marshaling of remotely-mapped Mapper outputs.

[ecables]

@otoolep -- The replication factor is 3 on a 3 node cluster, so the expectation is that everything is fully replicated.

[otoolep]

Can you show us the output of SHOW SHARDS. E.g.

$ ./influx 
Connected to http://localhost:8086 version 0.9
InfluxDB shell 0.9
> SHOW SHARDS
name: _internal
---------------
id      start_time              end_time                expiry_time             owners
1       2015-09-19T00:00:00Z    2015-09-20T00:00:00Z    2015-09-27T00:00:00Z    1

[ecables]

Here's output from one node in the cluster:

$ /opt/influxdb/influx
Connected to http://localhost:8086 version 0.9.4.1
InfluxDB shell 0.9.4.1
> show shards
name: stuffhere
-------------
id  start_time      end_time        expiry_time     owners
2   2015-08-17T00:00:00Z    2015-08-24T00:00:00Z    2017-08-23T00:00:00Z    3,1,2
4   2015-08-24T00:00:00Z    2015-08-31T00:00:00Z    2017-08-30T00:00:00Z    2,3,1
12  2015-08-31T00:00:00Z    2015-09-07T00:00:00Z    2017-09-06T00:00:00Z    2,3,1
20  2015-09-07T00:00:00Z    2015-09-14T00:00:00Z    2017-09-13T00:00:00Z    3,1,2
32  2015-09-14T00:00:00Z    2015-09-21T00:00:00Z    2017-09-20T00:00:00Z    2,3,1
54  2015-09-21T00:00:00Z    2015-09-28T00:00:00Z    2017-09-27T00:00:00Z    2,3,1
1   2015-08-22T00:00:00Z    2015-08-23T00:00:00Z    2015-10-07T00:00:00Z    2,3,1
3   2015-08-23T00:00:00Z    2015-08-24T00:00:00Z    2015-10-08T00:00:00Z    1,2,3
5   2015-08-24T00:00:00Z    2015-08-25T00:00:00Z    2015-10-09T00:00:00Z    3,1,2
6   2015-08-25T00:00:00Z    2015-08-26T00:00:00Z    2015-10-10T00:00:00Z    1,2,3
7   2015-08-26T00:00:00Z    2015-08-27T00:00:00Z    2015-10-11T00:00:00Z    2,3,1
8   2015-08-27T00:00:00Z    2015-08-28T00:00:00Z    2015-10-12T00:00:00Z    2,3,1
9   2015-08-28T00:00:00Z    2015-08-29T00:00:00Z    2015-10-13T00:00:00Z    3,1,2
10  2015-08-29T00:00:00Z    2015-08-30T00:00:00Z    2015-10-14T00:00:00Z    3,1,2
11  2015-08-30T00:00:00Z    2015-08-31T00:00:00Z    2015-10-15T00:00:00Z    3,1,2
13  2015-08-31T00:00:00Z    2015-09-01T00:00:00Z    2015-10-16T00:00:00Z    1,2,3
14  2015-09-01T00:00:00Z    2015-09-02T00:00:00Z    2015-10-17T00:00:00Z    2,3,1
15  2015-09-02T00:00:00Z    2015-09-03T00:00:00Z    2015-10-18T00:00:00Z    3,1,2
16  2015-09-03T00:00:00Z    2015-09-04T00:00:00Z    2015-10-19T00:00:00Z    2,3,1
17  2015-09-04T00:00:00Z    2015-09-05T00:00:00Z    2015-10-20T00:00:00Z    2,3,1
18  2015-09-05T00:00:00Z    2015-09-06T00:00:00Z    2015-10-21T00:00:00Z    1,2,3
19  2015-09-06T00:00:00Z    2015-09-07T00:00:00Z    2015-10-22T00:00:00Z    2,3,1
21  2015-09-07T00:00:00Z    2015-09-08T00:00:00Z    2015-10-23T00:00:00Z    1,2,3
22  2015-09-08T00:00:00Z    2015-09-09T00:00:00Z    2015-10-24T00:00:00Z    2,3,1
23  2015-09-09T00:00:00Z    2015-09-10T00:00:00Z    2015-10-25T00:00:00Z    1,2,3
24  2015-09-10T00:00:00Z    2015-09-11T00:00:00Z    2015-10-26T00:00:00Z    2,3,1
26  2015-09-11T00:00:00Z    2015-09-12T00:00:00Z    2015-10-27T00:00:00Z    1,2,3
28  2015-09-12T00:00:00Z    2015-09-13T00:00:00Z    2015-10-28T00:00:00Z    2,3,1
30  2015-09-13T00:00:00Z    2015-09-14T00:00:00Z    2015-10-29T00:00:00Z    3,1,2
33  2015-09-14T00:00:00Z    2015-09-15T00:00:00Z    2015-10-30T00:00:00Z    2,3,1
35  2015-09-15T00:00:00Z    2015-09-16T00:00:00Z    2015-10-31T00:00:00Z    1,2,3
37  2015-09-16T00:00:00Z    2015-09-17T00:00:00Z    2015-11-01T00:00:00Z    2,3,1
39  2015-09-17T00:00:00Z    2015-09-18T00:00:00Z    2015-11-02T00:00:00Z    1,2,3
41  2015-09-18T00:00:00Z    2015-09-19T00:00:00Z    2015-11-03T00:00:00Z    1,2,3
50  2015-09-19T00:00:00Z    2015-09-20T00:00:00Z    2015-11-04T00:00:00Z    1,2,3
52  2015-09-20T00:00:00Z    2015-09-21T00:00:00Z    2015-11-05T00:00:00Z    1,2,3
55  2015-09-21T00:00:00Z    2015-09-22T00:00:00Z    2015-11-06T00:00:00Z    1,2,3


name: _internal
---------------
id  start_time      end_time        expiry_time     owners
47  2015-09-18T00:00:00Z    2015-09-19T00:00:00Z    2015-09-26T00:00:00Z    1
48  2015-09-18T00:00:00Z    2015-09-19T00:00:00Z    2015-09-26T00:00:00Z    2
49  2015-09-18T00:00:00Z    2015-09-19T00:00:00Z    2015-09-26T00:00:00Z    3
51  2015-09-19T00:00:00Z    2015-09-20T00:00:00Z    2015-09-27T00:00:00Z    2,3,1
53  2015-09-20T00:00:00Z    2015-09-21T00:00:00Z    2015-09-28T00:00:00Z    2,3,1
56  2015-09-21T00:00:00Z    2015-09-22T00:00:00Z    2015-09-29T00:00:00Z    2,3,1

>

[otoolep]

OK, thanks, this confirms that the shards should be fully replicated. We're continuing to look into the issue here.

[otoolep]

I believe I see what is going on here, with regard to the specific panic. I am going to start on a fix now, hope to have this fixed very soon.

[otoolep]

Should be fixed by #4191